http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/61ba656f/content/documentation/runners/capability-matrix/index.html ---------------------------------------------------------------------- diff --git a/content/documentation/runners/capability-matrix/index.html b/content/documentation/runners/capability-matrix/index.html new file mode 100644 index 0000000..8a33334 --- /dev/null +++ b/content/documentation/runners/capability-matrix/index.html @@ -0,0 +1,1729 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache Beam Capability Matrix</title> + <meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes. +"> + + <link rel="stylesheet" href="/styles/site.css"> + <link rel="stylesheet" href="/css/theme.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <script src="/js/language-switch.js"></script> + <link rel="canonical" href="http://beam.incubator.apache.org/documentation/runners/capability-matrix/"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache Beam (incubating)" href="http://beam.incubator.apache.org/feed.xml";> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-73650088-1', 'auto'); + ga('send', 'pageview'); + + </script> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/get-started/beam-overview/">Beam Overview</a></li> + <li><a href="/get-started/quickstart/">Quickstart</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Example Walkthroughs</li> + <li><a href="/get-started/wordcount-example/">WordCount</a></li> + <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Resources</li> + <li><a href="/get-started/downloads">Downloads</a></li> + <li><a href="/get-started/releases">Release Notes</a></li> + <li><a href="/get-started/support">Support</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/documentation">Using the Documentation</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Beam Concepts</li> + <li><a href="/documentation/programming-guide/">Programming Guide</a></li> + <li><a href="/documentation/resources/">Additional Resources</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Pipeline Fundamentals</li> + <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li> + <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li> + <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">SDKs</li> + <li><a href="/documentation/sdks/java/">Java SDK</a></li> + <li><a href="/documentation/sdks/javadoc/">Java SDK API Reference</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Runners</li> + <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li> + <li><a href="/documentation/runners/direct/">Direct Runner</a></li> + <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li> + <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li> + <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/contribute">Get Started Contributing</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Guides</li> + <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li> + <li><a href="/contribute/testing/">Testing Guide</a></li> + <li><a href="/contribute/release-guide/">Release Guide</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Technical References</li> + <li><a href="/contribute/design-principles/">Design Principles</a></li> + <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li> + <li><a href="/contribute/source-repository/">Source Repository</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Promotion</li> + <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li> + <li><a href="/contribute/logos/">Logos and Design</a></li> + <li role="separator" class="divider"></li> + <li><a href="/contribute/team/">Team</a></li> + </ul> + </li> + + <li><a href="/blog">Blog</a></li> + </ul> + <ul class="nav navbar-nav navbar-right"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><img src="https://www.apache.org/foundation/press/kit/feather_small.png"; alt="Apache Logo" style="height:24px;">Apache Software Foundation<span class="caret"></span></a> + <ul class="dropdown-menu dropdown-menu-right"> + <li><a href="http://www.apache.org/";>ASF Homepage</a></li> + <li><a href="http://www.apache.org/licenses/";>License</a></li> + <li><a href="http://www.apache.org/security/";>Security</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html";>Thanks</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html";>Sponsorship</a></li> + <li><a href="https://www.apache.org/foundation/policies/conduct";>Code of Conduct</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + <h1 id="beam-capability-matrix">Beam Capability Matrix</h1> +<p><span style="font-size:11px;float:none">Last updated: 2016-11-04 16:45 PDT</span></p> + +<p>Apache Beam (incubating) provides a portable API layer for building sophisticated data-parallel processing engines that may be executed across a diversity of exeuction engines, or <i>runners</i>. The core concepts of this layer are based upon the Beam Model (formerly referred to as the <a href="http://www.vldb.org/pvldb/vol8/p1792-Akidau.pdf";>Dataflow Model</a>), and implemented to varying degrees in each Beam runner. To help clarify the capabilities of individual runners, weâve created the capability matrix below.</p> + +<p>Individual capabilities have been grouped by their corresponding <span class="wwwh-what-dark">What</span> / <span class="wwwh-where-dark">Where</span> / <span class="wwwh-when-dark">When</span> / <span class="wwwh-how-dark">How</span> question:</p> + +<ul> + <li><span class="wwwh-what-dark">What</span> results are being calculated?</li> + <li><span class="wwwh-where-dark">Where</span> in event time?</li> + <li><span class="wwwh-when-dark">When</span> in processing time?</li> + <li><span class="wwwh-how-dark">How</span> do refinements of results relate?</li> +</ul> + +<p>For more details on the <span class="wwwh-what-dark">What</span> / <span class="wwwh-where-dark">Where</span> / <span class="wwwh-when-dark">When</span> / <span class="wwwh-how-dark">How</span> breakdown of concepts, we recommend reading through the <a href="http://oreilly.com/ideas/the-world-beyond-batch-streaming-102";>Streaming 102</a> post on OâReilly Radar.</p> + +<p>Note that in the future, we intend to add additional tables beyond the current set, for things like runtime characterstics (e.g. at-least-once vs exactly-once), performance, etc.</p> + +<script type="text/javascript"> + function ToggleTables(showDetails, anchor) { + document.getElementById("cap-summary").style.display = showDetails ? "none" : "block"; + document.getElementById("cap-full").style.display = showDetails ? "block" : "none"; + location.hash = anchor; + } +</script> + +<center> + +<!-- Summary table --> + + + + + + +<div id="cap-summary" style="display:block"> +<table class="cap-summary"> + + <tr class="cap-summary" id="cap-summary-what"> + <th class="cap-summary color-metadata format-category" colspan="5" style="color:#ca1"><div class="cap-toggle" onclick="ToggleTables(1, "cap-full-what")">(expand details)</div>What is being computed?</th> + </tr> + <tr class="cap-summary"> + <th class="cap-summary color-capability"></th> + + <th class="cap-summary color-platform format-platform" style="color:#ec3">Beam Model</th> + + <th class="cap-summary color-platform format-platform" style="color:#ec3">Google Cloud Dataflow</th> + + <th class="cap-summary color-platform format-platform" style="color:#ec3">Apache Flink</th> + + <th class="cap-summary color-platform format-platform" style="color:#ec3">Apache Spark</th> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">ParDo</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">GroupByKey</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Flatten</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Combine</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Composite Transforms</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Side Inputs</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Source API</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Aggregators</th> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#ec3">Keyed State</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕ (<a href="https://issues.apache.org/jira/browse/BEAM-25";>BEAM-25</a>)</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <td class="cap-summary color-blank cap-blank" colspan="5"></td> + </tr> + + <tr class="cap-summary" id="cap-summary-where"> + <th class="cap-summary color-metadata format-category" colspan="5" style="color:#37d"><div class="cap-toggle" onclick="ToggleTables(1, "cap-full-where")">(expand details)</div>Where in event time?</th> + </tr> + <tr class="cap-summary"> + <th class="cap-summary color-capability"></th> + + <th class="cap-summary color-platform format-platform" style="color:#59f">Beam Model</th> + + <th class="cap-summary color-platform format-platform" style="color:#59f">Google Cloud Dataflow</th> + + <th class="cap-summary color-platform format-platform" style="color:#59f">Apache Flink</th> + + <th class="cap-summary color-platform format-platform" style="color:#59f">Apache Spark</th> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Global windows</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Fixed windows</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8cf;border-color:#37d"><center><b>~</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Sliding windows</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8cf;border-color:#37d"><center><b>~</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Session windows</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Custom windows</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Custom merging windows</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#59f">Timestamp control</th> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <td class="cap-summary color-blank cap-blank" colspan="5"></td> + </tr> + + <tr class="cap-summary" id="cap-summary-when"> + <th class="cap-summary color-metadata format-category" colspan="5" style="color:#6a4"><div class="cap-toggle" onclick="ToggleTables(1, "cap-full-when")">(expand details)</div>When in processing time?</th> + </tr> + <tr class="cap-summary"> + <th class="cap-summary color-capability"></th> + + <th class="cap-summary color-platform format-platform" style="color:#8c6">Beam Model</th> + + <th class="cap-summary color-platform format-platform" style="color:#8c6">Google Cloud Dataflow</th> + + <th class="cap-summary color-platform format-platform" style="color:#8c6">Apache Flink</th> + + <th class="cap-summary color-platform format-platform" style="color:#8c6">Apache Spark</th> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Configurable triggering</th> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Event-time triggers</th> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Processing-time triggers</th> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Count triggers</th> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">[Meta]data driven triggers</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕ (<a href="https://issues.apache.org/jira/browse/BEAM-101";>BEAM-101</a>)</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Composite triggers</th> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Allowed lateness</th> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#8c6">Timers</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕ (<a href="https://issues.apache.org/jira/browse/BEAM-27";>BEAM-27</a>)</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <td class="cap-summary color-blank cap-blank" colspan="5"></td> + </tr> + + <tr class="cap-summary" id="cap-summary-how"> + <th class="cap-summary color-metadata format-category" colspan="5" style="color:#b55"><div class="cap-toggle" onclick="ToggleTables(1, "cap-full-how")">(expand details)</div>How do refinements relate?</th> + </tr> + <tr class="cap-summary"> + <th class="cap-summary color-capability"></th> + + <th class="cap-summary color-platform format-platform" style="color:#d77">Beam Model</th> + + <th class="cap-summary color-platform format-platform" style="color:#d77">Google Cloud Dataflow</th> + + <th class="cap-summary color-platform format-platform" style="color:#d77">Apache Flink</th> + + <th class="cap-summary color-platform format-platform" style="color:#d77">Apache Spark</th> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#d77">Discarding</th> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#d77">Accumulating</th> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <th class="cap-summary color-capability format-capability" style="color:#d77">Accumulating & Retracting</th> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕ (<a href="https://issues.apache.org/jira/browse/BEAM-91";>BEAM-91</a>)</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> +</td> + + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> +</td> + + </tr> + + <tr class="cap-summary"> + <td class="cap-summary color-blank cap-blank" colspan="5"></td> + </tr> + +</table> +</div> + + +<!-- Full details table --> + + + + + + +<div id="cap-full" style="display:none"> +<table class="cap"> + + <tr class="cap" id="cap-full-what"> + <th class="cap color-metadata format-category" colspan="5" style="color:#ca1"><div class="cap-toggle" onclick="ToggleTables(0, "cap-summary-what")">(collapse details)</div>What is being computed?</th> + </tr> + <tr class="cap"> + <th class="cap color-capability"></th> + + <th class="cap color-platform format-platform" style="color:#ec3">Beam Model</th> + + <th class="cap color-platform format-platform" style="color:#ec3">Google Cloud Dataflow</th> + + <th class="cap color-platform format-platform" style="color:#ec3">Apache Flink</th> + + <th class="cap color-platform format-platform" style="color:#ec3">Apache Spark</th> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">ParDo</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: element-wise processing</b></center><br />Element-wise transformation parameterized by a chunk of user code. Elements are processed in bundles, with initialization and termination hooks. Bundle size is chosen by the runner and cannot be controlled by user code. ParDo processes a main input PCollection one element at a time, but provides side input access to additional PCollections. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Batch mode uses large bundle sizes. Streaming uses smaller bundle sizes. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />ParDo itself, as per-element transformation with UDFs, is fully supported by Flink for both batch and streaming. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />ParDo applies per-element transformations as Spark FlatMapFunction. +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">GroupByKey</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: key grouping</b></center><br />Grouping of key-value pairs per key, window, and pane. (See also other tabs.) +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Uses Flink's keyBy for key grouping. When grouping by window in streaming (creating the panes) the Flink runner uses the Beam code. This guarantees support for all windowing and triggering mechanisms. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: group by window in batch only</b></center><br />Uses Spark's groupByKey for grouping. Grouping by window is currently only supported in batch. +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Flatten</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: collection concatenation</b></center><br />Concatenates multiple homogenously typed collections together. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Combine</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: associative & commutative aggregation</b></center><br />Application of an associative, commutative operation over all values ("globally") or over all values associated with each key ("per key"). Can be implemented using ParDo, but often more efficient implementations exist. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: efficient execution</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Uses a combiner for pre-aggregation for batch and streaming. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Supports GroupedValues, Globally and PerKey. +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Composite Transforms</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: user-defined transformation subgraphs</b></center><br />Allows easy extensibility for library writers. In the near future, we expect there to be more information provided at this level -- customized metadata hooks for monitoring, additional runtime/environment hooks, etc. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br />Currently composite transformations are inlined during execution. The structure is later recreated from the names, but other transform level information (if added to the model) will be lost. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Side Inputs</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: additional elements available during DoFn execution</b></center><br />Side inputs are additional <tt>PCollections</tt> whose contents are computed during pipeline execution and then made accessible to DoFn code. The exact shape of the side input depends both on the <tt>PCollectionView</tt> used to describe the access pattern (interable, map, singleton) and the window of the element from the main input that is currently being processed. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: some size restrictions in streaming</b></center><br />Batch mode supports a distributed implementation, but streaming mode may force some size restrictions. Neither mode is able to push lookups directly up into key-based sources. +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: some size restrictions in streaming</b></center><br />Batch mode supports a distributed implementation, but streaming mode may force some size restrictions. Neither mode is able to push lookups directly up into key-based sources. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: not supported in streaming</b></center><br />Side input is actually a broadcast variable in Spark so it can't be updated during the life of a job. Spark-runner implementation of side input is more of an immutable, static, side input. +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Source API</th> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: user-defined sources</b></center><br />Allows users to provide additional input sources. Supports both bounded and unbounded data. Includes hooks necessary to provide efficient parallelization (size estimation, progress information, dynamic splitting, etc). +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Aggregators</th> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: user-provided metrics</b></center><br />Allow transforms to aggregate simple metrics across bundles in a <tt>DoFn</tt>. Semantically equivalent to using a side output, but support partial results as the transform executes. Will likely want to augment <tt>Aggregators</tt> to be more useful for processing unbounded data by making them windowed. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: may miscount in streaming mode</b></center><br />Current model is fully supported in batch mode. In streaming mode, <tt>Aggregators</tt> may under or overcount when bundles are retried. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: may undercount in streaming</b></center><br />Current model is fully supported in batch. In streaming mode, <tt>Aggregators</tt> may undercount. +</td> + + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: streaming requires more testing</b></center><br />Uses Spark's <tt>AccumulatorParam</tt> mechanism +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#ec3">Keyed State</th> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: storage per key, per window<br />(<a href="https://issues.apache.org/jira/browse/BEAM-25";>BEAM-25</a>)</b></center><br />Allows fine-grained access to per-key, per-window persistent state. Necessary for certain use cases (e.g. high-volume windows which store large amounts of data, but typically only access small portions of it; complex state machines; etc.) that are not easily or efficiently addressed via <tt>Combine</tt> or <tt>GroupByKey</tt>+<tt>ParDo</tt>. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: pending model support</b></center><br />Dataflow already supports keyed state internally, so adding support for this should be easy once the Beam model exposes it. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: pending model support</b></center><br />Flink already supports keyed state, so adding support for this should be easy once the Beam model exposes it. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: pending model support</b></center><br />Spark supports keyed state with mapWithState() so support shuold be straight forward. +</td> + + </tr> + + <tr class="cap"> + <td class="cap color-blank cap-blank" colspan="5"></td> + </tr> + + <tr class="cap" id="cap-full-where"> + <th class="cap color-metadata format-category" colspan="5" style="color:#37d"><div class="cap-toggle" onclick="ToggleTables(0, "cap-summary-where")">(collapse details)</div>Where in event time?</th> + </tr> + <tr class="cap"> + <th class="cap color-capability"></th> + + <th class="cap color-platform format-platform" style="color:#59f">Beam Model</th> + + <th class="cap color-platform format-platform" style="color:#59f">Google Cloud Dataflow</th> + + <th class="cap color-platform format-platform" style="color:#59f">Apache Flink</th> + + <th class="cap color-platform format-platform" style="color:#59f">Apache Spark</th> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Global windows</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: all time</b></center><br />The default window which covers all of time. (Basically how traditional batch cases fit in the model.) +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: default</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Fixed windows</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: periodic, non-overlapping</b></center><br />Fixed-size, timestamp-based windows. (Hourly, Daily, etc) +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: built-in</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#8cf;border-color:#37d"><center><b>Partially: currently only supported in batch</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Sliding windows</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: periodic, overlapping</b></center><br />Possibly overlapping fixed-size timestamp-based windows (Every minute, use the last ten minutes of data.) +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: built-in</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#8cf;border-color:#37d"><center><b>Partially: currently only supported in batch</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Session windows</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: activity-based</b></center><br />Based on bursts of activity separated by a gap size. Different per key. +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: built-in</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#37d"><center><b>No: pending Spark engine support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Custom windows</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: user-defined windows</b></center><br />All windows must implement <tt>BoundedWindow</tt>, which specifies a max timestamp. Each <tt>WindowFn</tt> assigns elements to an associated window. +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#37d"><center><b>No: pending Spark engine support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Custom merging windows</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: user-defined merging windows</b></center><br />A custom <tt>WindowFn</tt> additionally specifies whether and how to merge windows. +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#37d"><center><b>No: pending Spark engine support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#59f">Timestamp control</th> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: output timestamp for window panes</b></center><br />For a grouping transform, such as GBK or Combine, an OutputTimeFn specifies (1) how to combine input timestamps within a window and (2) how to merge aggregated timestamps when windows merge. +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#37d"><center><b>No: pending Spark engine support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <td class="cap color-blank cap-blank" colspan="5"></td> + </tr> + + <tr class="cap" id="cap-full-when"> + <th class="cap color-metadata format-category" colspan="5" style="color:#6a4"><div class="cap-toggle" onclick="ToggleTables(0, "cap-summary-when")">(collapse details)</div>When in processing time?</th> + </tr> + <tr class="cap"> + <th class="cap color-capability"></th> + + <th class="cap color-platform format-platform" style="color:#8c6">Beam Model</th> + + <th class="cap color-platform format-platform" style="color:#8c6">Google Cloud Dataflow</th> + + <th class="cap color-platform format-platform" style="color:#8c6">Apache Flink</th> + + <th class="cap color-platform format-platform" style="color:#8c6">Apache Spark</th> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Configurable triggering</th> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: user customizable</b></center><br />Triggering may be specified by the user (instead of simply driven by hardcoded defaults). +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />Fully supported in streaming mode. In batch mode, intermediate trigger firings are effectively meaningless. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Event-time triggers</th> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: relative to event time</b></center><br />Triggers that fire in response to event-time completeness signals, such as watermarks progressing. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: yes in streaming, fixed granularity in batch</b></center><br />Fully supported in streaming mode. In batch mode, currently watermark progress jumps from the beginning of time to the end of time once the input has been fully consumed, thus no additional triggering granularity is available. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Processing-time triggers</th> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: relative to processing time</b></center><br />Triggers that fire in response to processing-time advancing. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: yes in streaming, fixed granularity in batch</b></center><br />Fully supported in streaming mode. In batch mode, from the perspective of triggers, processing time currently jumps from the beginning of time to the end of time once the input has been fully consumed, thus no additional triggering granularity is available. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: This is Spark streaming's native model</b></center><br />Spark processes streams in micro-batches. The micro-batch size is actually a pre-set, fixed, time interval. Currently, the runner takes the first window size in the pipeline and sets it's size as the batch interval. Any following window operations will be considered processing time windows and will affect triggering. +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Count triggers</th> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: every N elements</b></center><br />Triggers that fire after seeing at least N elements. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />Fully supported in streaming mode. In batch mode, elements are processed in the largest bundles possible, so count-based triggers are effectively meaningless. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">[Meta]data driven triggers</th> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: in response to data<br />(<a href="https://issues.apache.org/jira/browse/BEAM-101";>BEAM-101</a>)</b></center><br />Triggers that fire in response to attributes of the data being processed. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Composite triggers</th> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: compositions of one or more sub-triggers</b></center><br />Triggers which compose other triggers in more complex structures, such as logical AND, logical OR, early/on-time/late, etc. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Allowed lateness</th> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: event-time bound on window lifetimes</b></center><br />A way to bound the useful lifetime of a window (in event time), after which any unemitted results may be materialized, the window contents may be garbage collected, and any addtional late data that arrive for the window may be discarded. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br />Fully supported in streaming mode. In batch mode no data is ever late. +</td> + + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#8c6">Timers</th> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: delayed processing callbacks<br />(<a href="https://issues.apache.org/jira/browse/BEAM-27";>BEAM-27</a>)</b></center><br />A fine-grained mechanism for performing work at some point in the future, in either the event-time or processing-time domain. Useful for orchestrating delayed events, timeouts, etc in complex state per-key, per-window state machines. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br />Dataflow already supports timers internally, so adding support for this should be easy once the Beam model exposes it. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br />Flink already supports timers internally, so adding support for this should be easy once the Beam model exposes it. +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <td class="cap color-blank cap-blank" colspan="5"></td> + </tr> + + <tr class="cap" id="cap-full-how"> + <th class="cap color-metadata format-category" colspan="5" style="color:#b55"><div class="cap-toggle" onclick="ToggleTables(0, "cap-summary-how")">(collapse details)</div>How do refinements relate?</th> + </tr> + <tr class="cap"> + <th class="cap color-capability"></th> + + <th class="cap color-platform format-platform" style="color:#d77">Beam Model</th> + + <th class="cap color-platform format-platform" style="color:#d77">Google Cloud Dataflow</th> + + <th class="cap color-platform format-platform" style="color:#d77">Apache Flink</th> + + <th class="cap color-platform format-platform" style="color:#d77">Apache Spark</th> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#d77">Discarding</th> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: panes discard elements when fired</b></center><br />Elements are discarded from accumulated state as their pane is fired. +</td> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br />Spark streaming natively discards elements after firing. +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#d77">Accumulating</th> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: panes accumulate elements across firings</b></center><br />Elements are accumulated in state across multiple pane firings for the same window. +</td> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br />Requires that the accumulated pane fits in memory, after being passed through the combiner (if relevant) +</td> + + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <th class="cap color-capability format-capability" style="color:#d77">Accumulating & Retracting</th> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: accumulation plus retraction of old panes<br />(<a href="https://issues.apache.org/jira/browse/BEAM-91";>BEAM-91</a>)</b></center><br />Elements are accumulated across multiple pane firings and old emitted values are retracted. Also known as "backsies" ;-D +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br /> +</td> + + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br /> +</td> + + </tr> + + <tr class="cap"> + <td class="cap color-blank cap-blank" colspan="5"></td> + </tr> + +</table> +</div> + +</center> + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved.</p> + <p class="text-center"><a href="/privacy_policy">Privacy Policy</a> | + <a href="/feed.xml">RSS Feed</a></p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + </body> + +</html>
http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/61ba656f/content/documentation/runners/dataflow/index.html ---------------------------------------------------------------------- diff --git a/content/documentation/runners/dataflow/index.html b/content/documentation/runners/dataflow/index.html new file mode 100644 index 0000000..32de918 --- /dev/null +++ b/content/documentation/runners/dataflow/index.html @@ -0,0 +1,166 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Cloud Dataflow Runner</title> + <meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes. +"> + + <link rel="stylesheet" href="/styles/site.css"> + <link rel="stylesheet" href="/css/theme.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <script src="/js/language-switch.js"></script> + <link rel="canonical" href="http://beam.incubator.apache.org/documentation/runners/dataflow/"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache Beam (incubating)" href="http://beam.incubator.apache.org/feed.xml";> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-73650088-1', 'auto'); + ga('send', 'pageview'); + + </script> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/get-started/beam-overview/">Beam Overview</a></li> + <li><a href="/get-started/quickstart/">Quickstart</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Example Walkthroughs</li> + <li><a href="/get-started/wordcount-example/">WordCount</a></li> + <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Resources</li> + <li><a href="/get-started/downloads">Downloads</a></li> + <li><a href="/get-started/releases">Release Notes</a></li> + <li><a href="/get-started/support">Support</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/documentation">Using the Documentation</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Beam Concepts</li> + <li><a href="/documentation/programming-guide/">Programming Guide</a></li> + <li><a href="/documentation/resources/">Additional Resources</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Pipeline Fundamentals</li> + <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li> + <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li> + <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">SDKs</li> + <li><a href="/documentation/sdks/java/">Java SDK</a></li> + <li><a href="/documentation/sdks/javadoc/">Java SDK API Reference</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Runners</li> + <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li> + <li><a href="/documentation/runners/direct/">Direct Runner</a></li> + <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li> + <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li> + <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/contribute">Get Started Contributing</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Guides</li> + <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li> + <li><a href="/contribute/testing/">Testing Guide</a></li> + <li><a href="/contribute/release-guide/">Release Guide</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Technical References</li> + <li><a href="/contribute/design-principles/">Design Principles</a></li> + <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li> + <li><a href="/contribute/source-repository/">Source Repository</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Promotion</li> + <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li> + <li><a href="/contribute/logos/">Logos and Design</a></li> + <li role="separator" class="divider"></li> + <li><a href="/contribute/team/">Team</a></li> + </ul> + </li> + + <li><a href="/blog">Blog</a></li> + </ul> + <ul class="nav navbar-nav navbar-right"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><img src="https://www.apache.org/foundation/press/kit/feather_small.png"; alt="Apache Logo" style="height:24px;">Apache Software Foundation<span class="caret"></span></a> + <ul class="dropdown-menu dropdown-menu-right"> + <li><a href="http://www.apache.org/";>ASF Homepage</a></li> + <li><a href="http://www.apache.org/licenses/";>License</a></li> + <li><a href="http://www.apache.org/security/";>Security</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html";>Thanks</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html";>Sponsorship</a></li> + <li><a href="https://www.apache.org/foundation/policies/conduct";>Code of Conduct</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + <h1 id="using-the-cloud-dataflow-runner">Using the Cloud Dataflow Runner</h1> + +<p>This page is under construction (<a href="https://issues.apache.org/jira/browse/BEAM-508";>BEAM-508</a>).</p> + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved.</p> + <p class="text-center"><a href="/privacy_policy">Privacy Policy</a> | + <a href="/feed.xml">RSS Feed</a></p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/61ba656f/content/documentation/runners/direct/index.html ---------------------------------------------------------------------- diff --git a/content/documentation/runners/direct/index.html b/content/documentation/runners/direct/index.html new file mode 100644 index 0000000..26e75df --- /dev/null +++ b/content/documentation/runners/direct/index.html @@ -0,0 +1,166 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache Direct Runner</title> + <meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes. +"> + + <link rel="stylesheet" href="/styles/site.css"> + <link rel="stylesheet" href="/css/theme.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <script src="/js/language-switch.js"></script> + <link rel="canonical" href="http://beam.incubator.apache.org/documentation/runners/direct/"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache Beam (incubating)" href="http://beam.incubator.apache.org/feed.xml";> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-73650088-1', 'auto'); + ga('send', 'pageview'); + + </script> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/get-started/beam-overview/">Beam Overview</a></li> + <li><a href="/get-started/quickstart/">Quickstart</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Example Walkthroughs</li> + <li><a href="/get-started/wordcount-example/">WordCount</a></li> + <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Resources</li> + <li><a href="/get-started/downloads">Downloads</a></li> + <li><a href="/get-started/releases">Release Notes</a></li> + <li><a href="/get-started/support">Support</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/documentation">Using the Documentation</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Beam Concepts</li> + <li><a href="/documentation/programming-guide/">Programming Guide</a></li> + <li><a href="/documentation/resources/">Additional Resources</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Pipeline Fundamentals</li> + <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li> + <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li> + <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">SDKs</li> + <li><a href="/documentation/sdks/java/">Java SDK</a></li> + <li><a href="/documentation/sdks/javadoc/">Java SDK API Reference</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Runners</li> + <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li> + <li><a href="/documentation/runners/direct/">Direct Runner</a></li> + <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li> + <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li> + <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/contribute">Get Started Contributing</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Guides</li> + <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li> + <li><a href="/contribute/testing/">Testing Guide</a></li> + <li><a href="/contribute/release-guide/">Release Guide</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Technical References</li> + <li><a href="/contribute/design-principles/">Design Principles</a></li> + <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li> + <li><a href="/contribute/source-repository/">Source Repository</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Promotion</li> + <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li> + <li><a href="/contribute/logos/">Logos and Design</a></li> + <li role="separator" class="divider"></li> + <li><a href="/contribute/team/">Team</a></li> + </ul> + </li> + + <li><a href="/blog">Blog</a></li> + </ul> + <ul class="nav navbar-nav navbar-right"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><img src="https://www.apache.org/foundation/press/kit/feather_small.png"; alt="Apache Logo" style="height:24px;">Apache Software Foundation<span class="caret"></span></a> + <ul class="dropdown-menu dropdown-menu-right"> + <li><a href="http://www.apache.org/";>ASF Homepage</a></li> + <li><a href="http://www.apache.org/licenses/";>License</a></li> + <li><a href="http://www.apache.org/security/";>Security</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html";>Thanks</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html";>Sponsorship</a></li> + <li><a href="https://www.apache.org/foundation/policies/conduct";>Code of Conduct</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + <h1 id="using-the-direct-runner">Using the Direct Runner</h1> + +<p>This page is under construction (<a href="https://issues.apache.org/jira/browse/BEAM-505";>BEAM-505</a>).</p> + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved.</p> + <p class="text-center"><a href="/privacy_policy">Privacy Policy</a> | + <a href="/feed.xml">RSS Feed</a></p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + </body> + +</html>
![https://www.apache.org/foundation/press/kit/feather_small.png"](https://www.apache.org/foundation/press/kit/feather_small.png"){kind=link}