http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/beam/capability/2016/03/17/capability-matrix.html ---------------------------------------------------------------------- diff --git a/content/beam/capability/2016/03/17/capability-matrix.html b/content/beam/capability/2016/03/17/capability-matrix.html index 3fd82e3..4d6ca5d 100644 --- a/content/beam/capability/2016/03/17/capability-matrix.html +++ b/content/beam/capability/2016/03/17/capability-matrix.html @@ -49,10 +49,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> @@ -150,23 +154,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> </tr> @@ -175,23 +175,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> </tr> @@ -200,23 +196,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> </tr> @@ -225,23 +217,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> </tr> @@ -250,23 +238,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> </tr> @@ -275,23 +259,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> </tr> @@ -300,23 +280,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"></td> </tr> @@ -325,23 +301,19 @@ - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"></td> </tr> @@ -350,23 +322,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"></td> </tr> @@ -395,23 +363,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> </tr> @@ -420,23 +384,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#8cf;border-color:#37d"><b><center>~</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8cf;border-color:#37d"></td> </tr> @@ -445,23 +405,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"></td> </tr> @@ -470,23 +426,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"></td> </tr> @@ -495,23 +447,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"></td> </tr> @@ -520,23 +468,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"></td> </tr> @@ -545,23 +489,19 @@ - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#37d"></td> </tr> @@ -590,23 +530,19 @@ - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -615,23 +551,19 @@ - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -640,23 +572,19 @@ - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> </tr> @@ -665,23 +593,19 @@ - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -690,23 +614,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -715,23 +635,19 @@ - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -740,23 +656,19 @@ - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -765,23 +677,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"></td> </tr> @@ -810,23 +718,19 @@ - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> </tr> @@ -835,23 +739,19 @@ - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><b><center>✓</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"></td> </tr> @@ -860,23 +760,19 @@ - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"></td> - <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><b><center>✕</center></b> -</td> + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"></td> </tr>
http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/beam/capability/2016/04/03/presentation-materials.html ---------------------------------------------------------------------- diff --git a/content/beam/capability/2016/04/03/presentation-materials.html b/content/beam/capability/2016/04/03/presentation-materials.html index 9b27c7d..5ee690d 100644 --- a/content/beam/capability/2016/04/03/presentation-materials.html +++ b/content/beam/capability/2016/04/03/presentation-materials.html @@ -49,10 +49,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> @@ -124,7 +128,7 @@ </li> </ul> -<p>As Apache Beam grows, so will this repository of presentation materials. We are excited to add new materials as the Apache Beam ecosystem grows with new runners, SDKs, and so on. If you are interested in contributing content or have a request, please see the <a href="/presentation-materials/">Apache Beam presentation materials</a> page or email the <a href="mailto:user@beam.incubator.apache.org"><code class="highlighter-rouge">[email protected]</code></a> mailing list with your ideas or questions.</p> +<p>As Apache Beam grows, so will this repository of presentation materials. We are excited to add new materials as the Apache Beam ecosystem grows with new runners, SDKs, and so on. If you are interested in contributing content or have a request, please see the <a href="/presentation-materials/">Apache Beam presentation materials</a> page or email the <a href="mailto:user@beam.incubator.apache.org"><code>[email protected]</code></a> mailing list with your ideas or questions.</p> </div> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/beam/python/sdk/2016/02/25/python-sdk-now-public.html ---------------------------------------------------------------------- diff --git a/content/beam/python/sdk/2016/02/25/python-sdk-now-public.html b/content/beam/python/sdk/2016/02/25/python-sdk-now-public.html index 62ffa48..0d09f7d 100644 --- a/content/beam/python/sdk/2016/02/25/python-sdk-now-public.html +++ b/content/beam/python/sdk/2016/02/25/python-sdk-now-public.html @@ -49,10 +49,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/beam/release/2016/06/15/first-release.html ---------------------------------------------------------------------- diff --git a/content/beam/release/2016/06/15/first-release.html b/content/beam/release/2016/06/15/first-release.html new file mode 100644 index 0000000..73a4159 --- /dev/null +++ b/content/beam/release/2016/06/15/first-release.html @@ -0,0 +1,165 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>The first release of Apache Beam!</title> + <meta name="description" content="Iâm happy to announce that Apache Beam has officially released its firstversion â 0.1.0-incubating. This is an exciting milestone for the project,which joine..."> + + <link rel="stylesheet" href="/styles/site.css"> + <link rel="stylesheet" href="/css/theme.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://beam.incubator.apache.org/beam/release/2016/06/15/first-release.html";> + <link rel="alternate" type="application/rss+xml" title="Apache Beam (incubating)" href="http://beam.incubator.apache.org/feed.xml";> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-73650088-1', 'auto'); + ga('send', 'pageview'); + + </script> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Guides</li> + <li><a href="/getting_started/">Getting Started</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> + <li><a href="/capability-matrix/">Capability Matrix</a></li> + <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> + <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header">Community</li> + <li><a href="/mailing_lists/">Mailing Lists</a></li> + <li><a href="/team/">Apache Beam Team</a></li> + <li><a href="/public-meetings/">Public Meetings</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Contribute</li> + <li><a href="/contribution-guide/">Contribution Guide</a></li> + <li><a href="/source_repository/">Source Repository</a></li> + <li><a href="/issue_tracking/">Issue Tracking</a></li> + </ul> + </li> + <li><a href="/blog">Blog</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Project <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/presentation-materials/">Presentation Materials</a></li> + <li><a href="/material/">Logos and design</a></li> + <li><a href="http://apache.org/licenses/LICENSE-2.0.html";>License</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="container"> + + +<article class="post" itemscope itemtype="http://schema.org/BlogPosting";> + + <header class="post-header"> + <h1 class="post-title" itemprop="name headline">The first release of Apache Beam!</h1> + <p class="post-meta"><time datetime="2016-06-15T00:00:01-07:00" itemprop="datePublished">Jun 15, 2016</time> ⢠Davor Bonaci [<a href="https://twitter.com/BonaciDavor";>@BonaciDavor</a>] +</p> + </header> + + <div class="post-content" itemprop="articleBody"> + <p>Iâm happy to announce that Apache Beam has officially released its first +version â 0.1.0-incubating. This is an exciting milestone for the project, +which joined the Apache Software Foundation and the Apache Incubator earlier +this year.</p> + +<!--more--> + +<p>This release publishes the first set of Apache Beam binaries and source code, +making them readily available for our users. The initial release includes the +SDK for Java, along with three runners: Apache Flink, Apache Spark and Google +Cloud Dataflow, a fully-managed cloud service. The release is available both +in the <a href="http://search.maven.org/#search%7Cga%7C1%7Cg%3A%22org.apache.beam%22";>Maven Central Repository</a>, +as well as a download from the <a href="/releases/">projectâs website</a>.</p> + +<p>The goal of this release was process-oriented. In particular, the Beam +community wanted to release existing functionality to our users, build and +validate the release processes, and obtain validation from the Apache Software +Foundation and the Apache Incubator.</p> + +<p>Iâd like to encourage everyone to try out this release. Please keep in mind +that this is the first incubating release â significant changes are to be +expected. As we march toward stability, a rapid cadence of future releases is +anticipated, perhaps one every 1-2 months.</p> + +<p>As always, the Beam community welcomes feedback. Stabilization, usability and +the developer experience will be our focus for the next several months. If you +have any comments or discover any issues, Iâd like to invite you to reach out +to us via <a href="/mailing_lists/">userâs mailing list</a> or the +<a href="https://issues.apache.org/jira/browse/BEAM/";>Apache JIRA issue tracker</a>.</p> + + </div> + +</article> + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved.</p> + <p class="text-center"><a href="/privacy_policy">Privacy Policy</a> | + <a href="/feed.xml">RSS Feed</a></p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/beam/update/website/2016/02/22/beam-has-a-logo.html ---------------------------------------------------------------------- diff --git a/content/beam/update/website/2016/02/22/beam-has-a-logo.html b/content/beam/update/website/2016/02/22/beam-has-a-logo.html index bd996dd..605cfda 100644 --- a/content/beam/update/website/2016/02/22/beam-has-a-logo.html +++ b/content/beam/update/website/2016/02/22/beam-has-a-logo.html @@ -49,10 +49,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> @@ -120,7 +124,7 @@ now has a logo.</p> unification of bath and streaming, as beams of light, within the âBâ. We will base our future website and documentation design around this logo and its coloring. We will also make various permutations and resolutions of this logo available in the -coming weeks. For any questions or comments, send an email to the <code class="highlighter-rouge">dev@</code> email list +coming weeks. For any questions or comments, send an email to the <code>dev@</code> email list for Apache Beam.</p> </div> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/blog/2016/05/18/splitAtFraction-method.html ---------------------------------------------------------------------- diff --git a/content/blog/2016/05/18/splitAtFraction-method.html b/content/blog/2016/05/18/splitAtFraction-method.html index d723556..d79956c 100644 --- a/content/blog/2016/05/18/splitAtFraction-method.html +++ b/content/blog/2016/05/18/splitAtFraction-method.html @@ -49,10 +49,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> @@ -110,7 +114,7 @@ <p>In a large batch processing job with many tasks executing in parallel, some of the tasks â the stragglers â can take a much longer time to complete than others, perhaps due to imperfect splitting of the work into parallel chunks when issuing the job. Typically, waiting for stragglers means that the overall job completes later than it should, and may also reserve too many machines that may be underutilized at the end. Cloud Dataflowâs dynamic work rebalancing can mitigate stragglers in most cases.</p> -<p>What Iâd like to highlight for the Apache Beam (incubating) community is that Cloud Dataflowâs dynamic work rebalancing is implemented using <em>runner-specific</em> control logic on top of Beamâs <em>runner-independent</em> <a href="https://github.com/apache/incubator-beam/blob/9fa97fb2491bc784df53fb0f044409dbbc2af3d7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java";><code class="highlighter-rouge">BoundedSource API</code></a>. Specifically, to steal work from a straggler, a runner need only call the readerâs <a href="https://github.com/apache/incubator-beam/blob/3edae9b8b4d7afefb5c803c19bb0a1c21ebba89d/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java#L266";><code class="highlighter-rouge">splitAtFraction method</code></a>. This will generate a new source containing leftover work, and then the runner can pass that source off to another idle worker. As Beam matures, I hope that other runners are interested in figuring out whether these APIs can help them improve performance, implementing dynamic work rebalancing, and collaborating on API changes that will help solve other pain points.</p> +<p>What Iâd like to highlight for the Apache Beam (incubating) community is that Cloud Dataflowâs dynamic work rebalancing is implemented using <em>runner-specific</em> control logic on top of Beamâs <em>runner-independent</em> <a href="https://github.com/apache/incubator-beam/blob/9fa97fb2491bc784df53fb0f044409dbbc2af3d7/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java";><code>BoundedSource API</code></a>. Specifically, to steal work from a straggler, a runner need only call the readerâs <a href="https://github.com/apache/incubator-beam/blob/3edae9b8b4d7afefb5c803c19bb0a1c21ebba89d/sdks/java/core/src/main/java/org/apache/beam/sdk/io/BoundedSource.java#L266";><code>splitAtFraction method</code></a>. This will generate a new source containing leftover work, and then the runner can pass that source off to another idle worker. As Beam matures, I hope that other runners are interested in figuring out whether these APIs can help them improve performance, imple menting dynamic work rebalancing, and collaborating on API changes that will help solve other pain points.</p> </div> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/blog/2016/05/27/where-is-my-pcollection-dot-map.html ---------------------------------------------------------------------- diff --git a/content/blog/2016/05/27/where-is-my-pcollection-dot-map.html b/content/blog/2016/05/27/where-is-my-pcollection-dot-map.html index 9f50244..606dec1 100644 --- a/content/blog/2016/05/27/where-is-my-pcollection-dot-map.html +++ b/content/blog/2016/05/27/where-is-my-pcollection-dot-map.html @@ -6,7 +6,7 @@ <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1"> - <title>Where's my PCollection.map()?</title> + <title>Where's my PCollection.map()?</title> <meta name="description" content="Have you ever wondered why Beam has PTransforms for everything instead of having methods on PCollection? Take a look at the history that led to this (and oth..."> <link rel="stylesheet" href="/styles/site.css"> @@ -49,10 +49,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> @@ -110,7 +114,7 @@ <p>Though Beam is relatively new, its design draws heavily on many years of experience with real-world pipelines. One of the primary inspirations is <a href="http://research.google.com/pubs/pub35650.html";>FlumeJava</a>, which is Googleâs internal successor to MapReduce first introduced in 2009.</p> -<p>The original FlumeJava API has methods like <code class="highlighter-rouge">count</code> and <code class="highlighter-rouge">parallelDo</code> on the PCollections. Though slightly more succinct, this approach has many disadvantages to extensibility. Every new user to FlumeJava wanted to add transforms, and adding them as methods to PCollection simply doesnât scale well. In contrast, a PCollection in Beam has a single <code class="highlighter-rouge">apply</code> method which takes any PTransform as an argument.</p> +<p>The original FlumeJava API has methods like <code>count</code> and <code>parallelDo</code> on the PCollections. Though slightly more succinct, this approach has many disadvantages to extensibility. Every new user to FlumeJava wanted to add transforms, and adding them as methods to PCollection simply doesnât scale well. In contrast, a PCollection in Beam has a single <code>apply</code> method which takes any PTransform as an argument.</p> <table class="table"> <tr> @@ -134,7 +138,7 @@ PCollection<O> output = input.apply(Count.perElement()) <p>This is a more scalable approach for several reasons.</p> <h2 id="where-to-draw-the-line">Where to draw the line?</h2> -<p>Adding methods to PCollection forces a line to be drawn between operations that are âusefulâ enough to merit this special treatment and those that are not. It is easy to make the case for flat map, group by key, and combine per key. But what about filter? Count? Approximate count? Approximate quantiles? Most frequent? WriteToMyFavoriteSource? Going too far down this path leads to a single enormous class that contains nearly everything one could want to do. (FlumeJavaâs PCollection class is over 5000 lines long with around 70 distinct operations, and it could have been <em>much</em> larger had we accepted every proposal.) Furthermore, since Java doesnât allow adding methods to a class, there is a sharp syntactic divide between those operations that are added to PCollection and those that arenât. A traditional way to share code is with a library of functions, but functions (in traditional languages like Java at least) are written prefix-style, which doesnât mix well wit h the fluent builder style (e.g. <code class="highlighter-rouge">input.operation1().operation2().operation3()</code> vs. <code class="highlighter-rouge">operation3(operation1(input).operation2())</code>).</p> +<p>Adding methods to PCollection forces a line to be drawn between operations that are âusefulâ enough to merit this special treatment and those that are not. It is easy to make the case for flat map, group by key, and combine per key. But what about filter? Count? Approximate count? Approximate quantiles? Most frequent? WriteToMyFavoriteSource? Going too far down this path leads to a single enormous class that contains nearly everything one could want to do. (FlumeJavaâs PCollection class is over 5000 lines long with around 70 distinct operations, and it could have been <em>much</em> larger had we accepted every proposal.) Furthermore, since Java doesnât allow adding methods to a class, there is a sharp syntactic divide between those operations that are added to PCollection and those that arenât. A traditional way to share code is with a library of functions, but functions (in traditional languages like Java at least) are written prefix-style, which doesnât mix well wit h the fluent builder style (e.g. <code>input.operation1().operation2().operation3()</code> vs. <code>operation3(operation1(input).operation2())</code>).</p> <p>Instead in Beam weâve chosen a style that places all transformsâwhether they be primitive operations, composite operations bundled in the SDK, or part of an external libraryâon equal footing. This also facilitates alternative implementations (which may even take different options) that are easily interchangeable.</p> @@ -162,14 +166,14 @@ PCollection<O> output = input </table> <h2 id="configurability">Configurability</h2> -<p>It makes for a fluent style to let values (PCollections) be the objects passed around and manipulated (i.e. the handles to the deferred execution graph), but it is the operations themselves that need to be composable, configurable, and extendable. Using PCollection methods for the operations doesnât scale well here, especially in a language without default or keyword arguments. For example, a ParDo operation can have any number of side inputs and side outputs, or a write operation may have configurations dealing with encoding and compression. One option is to separate these out into multiple overloads or even methods, but that exacerbates the problems above. (FlumeJava evolved over a dozen overloads of the <code class="highlighter-rouge">parallelDo</code> method!) Another option is to pass each method a configuration object that can be built up using more fluent idioms like the builder pattern, but at that point one might as well make the configuration object the operation itse lf, which is what Beam does.</p> +<p>It makes for a fluent style to let values (PCollections) be the objects passed around and manipulated (i.e. the handles to the deferred execution graph), but it is the operations themselves that need to be composable, configurable, and extendable. Using PCollection methods for the operations doesnât scale well here, especially in a language without default or keyword arguments. For example, a ParDo operation can have any number of side inputs and side outputs, or a write operation may have configurations dealing with encoding and compression. One option is to separate these out into multiple overloads or even methods, but that exacerbates the problems above. (FlumeJava evolved over a dozen overloads of the <code>parallelDo</code> method!) Another option is to pass each method a configuration object that can be built up using more fluent idioms like the builder pattern, but at that point one might as well make the configuration object the operation itself, which is what Beam doe s.</p> <h2 id="type-safety">Type Safety</h2> -<p>Many operations can only be applied to collections whose elements are of a specific type. For example, the GroupByKey operation should only be applied to <code class="highlighter-rouge">PCollection<KV<K, V>></code>s. In Java at least, itâs not possible to restrict methods based on the element type parameter alone. In FlumeJava, this led us to add a <code class="highlighter-rouge">PTable<K, V></code> subclassing <code class="highlighter-rouge">PCollection<KV<K, V>></code> to contain all the operations specific to PCollections of key-value pairs. This leads to the same question of which element types are special enough to merit being captured by PCollection subclasses. It is not very extensible for third parties and often requires manual downcasts/conversions (which canât be safely chained in Java) and special operations that produce these PCollection specializations.</p> +<p>Many operations can only be applied to collections whose elements are of a specific type. For example, the GroupByKey operation should only be applied to <code>PCollection<KV<K, V>></code>s. In Java at least, itâs not possible to restrict methods based on the element type parameter alone. In FlumeJava, this led us to add a <code>PTable<K, V></code> subclassing <code>PCollection<KV<K, V>></code> to contain all the operations specific to PCollections of key-value pairs. This leads to the same question of which element types are special enough to merit being captured by PCollection subclasses. It is not very extensible for third parties and often requires manual downcasts/conversions (which canât be safely chained in Java) and special operations that produce these PCollection specializations.</p> <p>This is particularly inconvenient for transforms that produce outputs whose element types are the same as (or related to) their inputâs element types, requiring extra support to generate the right subclasses (e.g. a filter on a PTable should produce another PTable rather than just a raw PCollection of key-value pairs).</p> -<p>Using PTransforms allows us to sidestep this entire issue. We can place arbitrary constraints on the context in which a transform may be used based on the type of its inputs; for instance GroupByKey is statically typed to only apply to a <code class="highlighter-rouge">PCollection<KV<K, V>></code>. The way this happens is generalizable to arbitrary shapes, without needing to introduce specialized types like PTable.</p> +<p>Using PTransforms allows us to sidestep this entire issue. We can place arbitrary constraints on the context in which a transform may be used based on the type of its inputs; for instance GroupByKey is statically typed to only apply to a <code>PCollection<KV<K, V>></code>. The way this happens is generalizable to arbitrary shapes, without needing to introduce specialized types like PTable.</p> <h2 id="reusability-and-structure">Reusability and Structure</h2> <p>Though PTransforms are generally constructed at the site at which theyâre used, by pulling them out as separate objects one is able to store them and pass them around.</p> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/blog/2016/06/13/flink-batch-runner-milestone.html ---------------------------------------------------------------------- diff --git a/content/blog/2016/06/13/flink-batch-runner-milestone.html b/content/blog/2016/06/13/flink-batch-runner-milestone.html new file mode 100644 index 0000000..c23f566 --- /dev/null +++ b/content/blog/2016/06/13/flink-batch-runner-milestone.html @@ -0,0 +1,159 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>How We Added Windowing to the Apache Flink Batch Runner</title> + <meta name="description" content="We recently achieved a major milestone by adding support for windowing to the Apache Flink Batch runner. In this post we would like to explain what this mean..."> + + <link rel="stylesheet" href="/styles/site.css"> + <link rel="stylesheet" href="/css/theme.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <link rel="canonical" href="http://beam.incubator.apache.org/blog/2016/06/13/flink-batch-runner-milestone.html";> + <link rel="alternate" type="application/rss+xml" title="Apache Beam (incubating)" href="http://beam.incubator.apache.org/feed.xml";> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-73650088-1', 'auto'); + ga('send', 'pageview'); + + </script> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Guides</li> + <li><a href="/getting_started/">Getting Started</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> + <li><a href="/capability-matrix/">Capability Matrix</a></li> + <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> + <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Community <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li class="dropdown-header">Community</li> + <li><a href="/mailing_lists/">Mailing Lists</a></li> + <li><a href="/team/">Apache Beam Team</a></li> + <li><a href="/public-meetings/">Public Meetings</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Contribute</li> + <li><a href="/contribution-guide/">Contribution Guide</a></li> + <li><a href="/source_repository/">Source Repository</a></li> + <li><a href="/issue_tracking/">Issue Tracking</a></li> + </ul> + </li> + <li><a href="/blog">Blog</a></li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Project <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/presentation-materials/">Presentation Materials</a></li> + <li><a href="/material/">Logos and design</a></li> + <li><a href="http://apache.org/licenses/LICENSE-2.0.html";>License</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="container"> + + +<article class="post" itemscope itemtype="http://schema.org/BlogPosting";> + + <header class="post-header"> + <h1 class="post-title" itemprop="name headline">How We Added Windowing to the Apache Flink Batch Runner</h1> + <p class="post-meta"><time datetime="2016-06-13T09:00:00-07:00" itemprop="datePublished">Jun 13, 2016</time> ⢠Aljoscha Krettek [<a href="https://twitter.com/aljoscha";>@aljoscha</a>] +</p> + </header> + + <div class="post-content" itemprop="articleBody"> + <p>We recently achieved a major milestone by adding support for windowing to the <a href="http://flink.apache.org";>Apache Flink</a> Batch runner. In this post we would like to explain what this means for users of Apache Beam and highlight some of the implementation details.</p> + +<!--more--> + +<p>Before we start, though, letâs quickly talk about the execution of Beam programs and how this is relevant to todayâs post. A Beam pipeline can contain bounded and unbounded sources. If the pipeline only contains bounded sources it can be executed in a batch fashion, if it contains some unbounded sources it must be executed in a streaming fashion. When executing a Beam pipeline on Flink, you donât have to choose the execution mode. Internally, the Flink runner either translates the pipeline to a Flink <code>DataSet</code> program or a <code>DataStream</code> program, depending on whether unbounded sources are used in the pipeline. In the following, when we say âBatch runnerâ what we are really talking about is the Flink runner being in batch execution mode.</p> + +<h2 id="what-does-this-mean-for-users">What does this mean for users?</h2> + +<p>Support for windowing was the last missing puzzle piece for making the Flink Batch runner compatible with the Beam model. With the latest change to the Batch runner users can now run any pipeline that only contains bounded sources and be certain that the results match those of the original reference-implementation runners that were provided by Google as part of the initial code drop coming from the Google Dataflow SDK.</p> + +<p>The most obvious part of the change is that windows can now be assigned to elements and that the runner respects these windows for the <code>GroupByKey</code> and <code>Combine</code> operations. A not-so-obvious change concerns side-inputs. In the Beam model, side inputs respect windows; when a value of the main input is being processed only the side input that corresponds to the correct window is available to the processing function, the <code>DoFn</code>.</p> + +<p>Getting side-input semantics right is an important milestone in itâs own because it allows to use a big suite of unit tests for verifying the correctness of a runner implementation. These tests exercise every obscure detail of the Beam programming model and verify that the results produced by a runner match what you would expect from a correct implementation. In the suite, side inputs are used to compare the expected result to the actual result. With these tests being executed regularly we can now be more confident that the implementation produces correct results for user-specified pipelines.</p> + +<h2 id="under-the-hood">Under the Hood</h2> +<p>The basis for the changes is the introduction of <code>WindowedValue</code> in the generated Flink transformations. Before, a Beam <code>PCollection<T></code> would be transformed to a <code>DataSet<T></code>. Now, we instead create a <code>DataSet<WindowedValue<T>></code>. The <code>WindowedValue<T></code> stores meta data about the value, such as the timestamp and the windows to which it was assigned.</p> + +<p>With this basic change out of the way we just had to make sure that windows were respected for side inputs and that <code>Combine</code> and <code>GroupByKey</code> correctly handled windows. The tricky part there is the handling of merging windows such as session windows. For these we essentially emulate the behavior of a merging <code>WindowFn</code> in our own code.</p> + +<p>After we got side inputs working we could enable the aforementioned suite of tests to check how well the runner behaves with respect to the Beam model. As can be expected there were quite some discrepancies but we managed to resolve them all. In the process, we also slimmed down the runner implementation. For example, we removed all custom translations for sources and sinks and are now relying only on Beam code for these, thereby greatly reducing the maintenance overhead.</p> + +<h2 id="summary">Summary</h2> +<p>We reached a major milestone in adding windowing support to the Flink Batch runner, thereby making it compatible with the Beam model. Because of the large suite of tests that can now be executed on the runner we are also confident about the correctness of the implementation and about it staying that way in the future.</p> + + </div> + +</article> + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center">© Copyright 2016 + <a href="http://www.apache.org";>The Apache Software Foundation.</a> All Rights Reserved.</p> + <p class="text-center"><a href="/privacy_policy">Privacy Policy</a> | + <a href="/feed.xml">RSS Feed</a></p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/incubator-beam-site/blob/6fce44a2/content/blog/index.html ---------------------------------------------------------------------- diff --git a/content/blog/index.html b/content/blog/index.html index ab91ae1..eee879d 100644 --- a/content/blog/index.html +++ b/content/blog/index.html @@ -50,10 +50,14 @@ <li class="dropdown"> <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> <ul class="dropdown-menu"> + <li class="dropdown-header">Releases</li> + <li><a href="/releases/">Release information</a></li> + <li role="separator" class="divider"></li> <li class="dropdown-header">Guides</li> <li><a href="/getting_started/">Getting Started</a></li> <li role="separator" class="divider"></li> <li class="dropdown-header">Technical Documentation</li> + <li><a href="/javadoc/0.1.0-incubating/">Java API Reference</a></li> <li><a href="/capability-matrix/">Capability Matrix</a></li> <li><a href="https://goo.gl/ps8twC";>Technical Docs</a></li> <li><a href="https://goo.gl/nk5OM0";>Technical Vision</a></li> @@ -104,6 +108,41 @@ <p>This is the blog for the Apache Beam project. This blog contains news and updates for the project.</p> +<h3 id="a-classpost-link-hrefbeamrelease20160615first-releasehtmlthe-first-release-of-apache-beama"><a class="post-link" href="/beam/release/2016/06/15/first-release.html">The first release of Apache Beam!</a></h3> +<p><i>Jun 15, 2016 ⢠Davor Bonaci [<a href="https://twitter.com/BonaciDavor";>@BonaciDavor</a>] +</i></p> + +<p>Iâm happy to announce that Apache Beam has officially released its first +version â 0.1.0-incubating. This is an exciting milestone for the project, +which joined the Apache Software Foundation and the Apache Incubator earlier +this year.</p> + +<!-- Render a "read more" button if the post is longer than the excerpt --> + +<p> +<a class="btn btn-default btn-sm" href="/beam/release/2016/06/15/first-release.html#read-more" role="button"> +Read more <span class="glyphicon glyphicon-menu-right" aria-hidden="true"></span> +</a> +</p> + +<hr /> + +<h3 id="a-classpost-link-hrefblog20160613flink-batch-runner-milestonehtmlhow-we-added-windowing-to-the-apache-flink-batch-runnera"><a class="post-link" href="/blog/2016/06/13/flink-batch-runner-milestone.html">How We Added Windowing to the Apache Flink Batch Runner</a></h3> +<p><i>Jun 13, 2016 ⢠Aljoscha Krettek [<a href="https://twitter.com/aljoscha";>@aljoscha</a>] +</i></p> + +<p>We recently achieved a major milestone by adding support for windowing to the <a href="http://flink.apache.org";>Apache Flink</a> Batch runner. In this post we would like to explain what this means for users of Apache Beam and highlight some of the implementation details.</p> + +<!-- Render a "read more" button if the post is longer than the excerpt --> + +<p> +<a class="btn btn-default btn-sm" href="/blog/2016/06/13/flink-batch-runner-milestone.html#read-more" role="button"> +Read more <span class="glyphicon glyphicon-menu-right" aria-hidden="true"></span> +</a> +</p> + +<hr /> + <h3 id="a-classpost-link-hrefblog20160527where-is-my-pcollection-dot-maphtmlwheres-my-pcollectionmapa"><a class="post-link" href="/blog/2016/05/27/where-is-my-pcollection-dot-map.html">Whereâs my PCollection.map()?</a></h3> <p><i>May 27, 2016 ⢠Robert Bradshaw </i></p> @@ -193,9 +232,6 @@ Read more <span class="glyphicon glyphicon-menu-right" aria-hidden="true">< both batch and stream processing into one powerful model. In fact, this unification is so important, the name Beam itself comes from the union of <strong>B</strong>atch + str<strong>EAM</strong> = Beam</p> -<p>When the project started, we wanted a logo which was both appealing and visually -represented this unification.</p> - <!-- Render a "read more" button if the post is longer than the excerpt --> <p>
