Regenerate website
Project: http://git-wip-us.apache.org/repos/asf/beam-site/repo Commit: http://git-wip-us.apache.org/repos/asf/beam-site/commit/d7f46849 Tree: http://git-wip-us.apache.org/repos/asf/beam-site/tree/d7f46849 Diff: http://git-wip-us.apache.org/repos/asf/beam-site/diff/d7f46849 Branch: refs/heads/asf-site Commit: d7f46849112abafd182eb107aca486e00df07087 Parents: da7d6c0 Author: Kenneth Knowles <[email protected]> Authored: Wed May 3 14:03:01 2017 -0700 Committer: Kenneth Knowles <[email protected]> Committed: Wed May 3 14:03:01 2017 -0700 ---------------------------------------------------------------------- content/contribute/work-in-progress/index.html | 2 +- .../runners/capability-matrix/index.html | 286 +++++++++++++++++ .../documentation/runners/gearpump/index.html | 313 +++++++++++++++++++ content/get-started/beam-overview/index.html | 12 +- content/images/logos/runners/gearpump.png | Bin 0 -> 2643 bytes 5 files changed, 611 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/contribute/work-in-progress/index.html ---------------------------------------------------------------------- diff --git a/content/contribute/work-in-progress/index.html b/content/contribute/work-in-progress/index.html index 90eff24..07abdf1 100644 --- a/content/contribute/work-in-progress/index.html +++ b/content/contribute/work-in-progress/index.html @@ -184,7 +184,7 @@ <td>Apache Gearpump Runner</td> <td><a href="https://github.com/apache/beam/tree/gearpump-runner";>gearpump-runner</a></td> <td><a href="https://issues.apache.org/jira/browse/BEAM/component/12330829";>runner-gearpump</a></td> - <td><a href="https://github.com/apache/beam/blob/gearpump-runner/runners/gearpump/README.md";>README</a></td> + <td><a href="/documentation/runners/gearpump/">runner homepage</a></td> </tr> <tr> <td>Apache Spark 2.0 Runner</td> http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/documentation/runners/capability-matrix/index.html ---------------------------------------------------------------------- diff --git a/content/documentation/runners/capability-matrix/index.html b/content/documentation/runners/capability-matrix/index.html index 3faf1f2..24fa451 100644 --- a/content/documentation/runners/capability-matrix/index.html +++ b/content/documentation/runners/capability-matrix/index.html @@ -203,6 +203,8 @@ <th class="cap-summary color-platform format-platform" style="color:#ec3">Apache Apex</th> + <th class="cap-summary color-platform format-platform" style="color:#ec3">Apache Gearpump</th> + </tr> <tr class="cap-summary"> @@ -233,6 +235,11 @@ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -263,6 +270,11 @@ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -293,6 +305,11 @@ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -323,6 +340,11 @@ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -353,6 +375,11 @@ <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#fe5;border-color:#ca1"><center><b>~</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -383,6 +410,11 @@ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -413,6 +445,11 @@ <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ec3;border-color:#ca1"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -443,6 +480,11 @@ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -473,6 +515,11 @@ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#ca1"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -495,6 +542,8 @@ <th class="cap-summary color-platform format-platform" style="color:#59f">Apache Apex</th> + <th class="cap-summary color-platform format-platform" style="color:#59f">Apache Gearpump</th> + </tr> <tr class="cap-summary"> @@ -525,6 +574,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -555,6 +609,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -585,6 +644,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -615,6 +679,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -645,6 +714,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -675,6 +749,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -705,6 +784,11 @@ <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#59f;border-color:#37d"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -727,6 +811,8 @@ <th class="cap-summary color-platform format-platform" style="color:#8c6">Apache Apex</th> + <th class="cap-summary color-platform format-platform" style="color:#8c6">Apache Gearpump</th> + </tr> <tr class="cap-summary"> @@ -757,6 +843,11 @@ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -787,6 +878,11 @@ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -817,6 +913,11 @@ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -847,6 +948,11 @@ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -877,6 +983,11 @@ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -907,6 +1018,11 @@ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -937,6 +1053,11 @@ <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#8c6;border-color:#6a4"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -967,6 +1088,11 @@ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#6a4"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -989,6 +1115,8 @@ <th class="cap-summary color-platform format-platform" style="color:#d77">Apache Apex</th> + <th class="cap-summary color-platform format-platform" style="color:#d77">Apache Gearpump</th> + </tr> <tr class="cap-summary"> @@ -1019,6 +1147,11 @@ <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -1049,6 +1182,11 @@ <td width="25%" class="cap-summary" style="background-color:#d77;border-color:#b55"><center><b>✓</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -1079,6 +1217,11 @@ <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> </td> + + + <td width="25%" class="cap-summary" style="background-color:#ddd;border-color:#b55"><center><b>✕</b></center> +</td> + </tr> <tr class="cap-summary"> @@ -1115,6 +1258,8 @@ <th class="cap color-platform format-platform" style="color:#ec3">Apache Apex</th> + <th class="cap color-platform format-platform" style="color:#ec3">Apache Gearpump</th> + </tr> <tr class="cap"> @@ -1145,6 +1290,11 @@ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Supported through Apex operator that wraps the function and processes data as single element bundles. </td> + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Gearpump wraps the per-element transformation function into processor execution. +</td> + </tr> <tr class="cap"> @@ -1175,6 +1325,11 @@ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Apex runner uses the Beam code for grouping by window and thereby has support for all windowing and triggering mechanisms. Runner does not implement partitioning yet (BEAM-838) </td> + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Use Gearpump's groupBy and window for key grouping and translate Beam's windowing and triggering to Gearpump's internal implementation. +</td> + </tr> <tr class="cap"> @@ -1205,6 +1360,11 @@ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1235,6 +1395,11 @@ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Default Beam translation. Currently no efficient pre-aggregation (BEAM-935). </td> + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1265,6 +1430,11 @@ <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#fe5;border-color:#ca1"><center><b>Partially: supported via inlining</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1295,6 +1465,11 @@ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: size restrictions</b></center><br />No distributed implementation and therefore size restrictions. </td> + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br />Implemented by merging side input as a normal stream in Gearpump +</td> + </tr> <tr class="cap"> @@ -1325,6 +1500,11 @@ <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ec3;border-color:#ca1"><center><b>Yes: fully supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1355,6 +1535,11 @@ <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: Not implemented in runner.</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No</b></center><br />not implemented +</td> + </tr> <tr class="cap"> @@ -1385,6 +1570,11 @@ <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: not implemented</b></center><br />Apex supports per-key state, so adding support for this should be easy. </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#ca1"><center><b>No: not implemented</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1407,6 +1597,8 @@ <th class="cap color-platform format-platform" style="color:#59f">Apache Apex</th> + <th class="cap color-platform format-platform" style="color:#59f">Apache Gearpump</th> + </tr> <tr class="cap"> @@ -1437,6 +1629,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1467,6 +1664,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1497,6 +1699,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1527,6 +1734,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1557,6 +1769,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1587,6 +1804,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1617,6 +1839,11 @@ <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#59f;border-color:#37d"><center><b>Yes: supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1639,6 +1866,8 @@ <th class="cap color-platform format-platform" style="color:#8c6">Apache Apex</th> + <th class="cap color-platform format-platform" style="color:#8c6">Apache Gearpump</th> + </tr> <tr class="cap"> @@ -1669,6 +1898,11 @@ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1699,6 +1933,11 @@ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1729,6 +1968,11 @@ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1759,6 +2003,11 @@ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1789,6 +2038,11 @@ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: pending model support</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1819,6 +2073,11 @@ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1849,6 +2108,11 @@ <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#8c6;border-color:#6a4"><center><b>Yes: fully supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1879,6 +2143,11 @@ <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: not implemented</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#6a4"><center><b>No: not implemented</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1901,6 +2170,8 @@ <th class="cap color-platform format-platform" style="color:#d77">Apache Apex</th> + <th class="cap color-platform format-platform" style="color:#d77">Apache Gearpump</th> + </tr> <tr class="cap"> @@ -1931,6 +2202,11 @@ <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1961,6 +2237,11 @@ <td width="25%" class="cap" style="background-color:#d77;border-color:#b55"><center><b>Yes: fully supported</b></center><br />Size restriction, see combine support. </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No</b></center><br /> +</td> + </tr> <tr class="cap"> @@ -1991,6 +2272,11 @@ <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br /> </td> + + + <td width="25%" class="cap" style="background-color:#ddd;border-color:#b55"><center><b>No: pending model support</b></center><br /> +</td> + </tr> <tr class="cap"> http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/documentation/runners/gearpump/index.html ---------------------------------------------------------------------- diff --git a/content/documentation/runners/gearpump/index.html b/content/documentation/runners/gearpump/index.html new file mode 100644 index 0000000..4a9fd82 --- /dev/null +++ b/content/documentation/runners/gearpump/index.html @@ -0,0 +1,313 @@ +<!DOCTYPE html> +<html lang="en"> + + <head> + <meta charset="utf-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + + <title>Apache Gearpump (incubating) Runner</title> + <meta name="description" content="Apache Beam is an open source, unified model and set of language-specific SDKs for defining and executing data processing workflows, and also data ingestion and integration flows, supporting Enterprise Integration Patterns (EIPs) and Domain Specific Languages (DSLs). Dataflow pipelines simplify the mechanics of large-scale batch and streaming data processing and can run on a number of runtimes like Apache Flink, Apache Spark, and Google Cloud Dataflow (a cloud service). Beam also brings DSL in different languages, allowing users to easily implement their data integration processes. +"> + + <link rel="stylesheet" href="/styles/site.css"> + <link rel="stylesheet" href="/css/theme.css"> + <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.2.0/jquery.min.js";></script> + <script src="/js/bootstrap.min.js"></script> + <script src="/js/language-switch.js"></script> + <link rel="canonical" href="https://beam.apache.org/documentation/runners/gearpump/"; data-proofer-ignore> + <link rel="alternate" type="application/rss+xml" title="Apache Beam" href="https://beam.apache.org/feed.xml";> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-73650088-1', 'auto'); + ga('send', 'pageview'); + + </script> + <link rel="shortcut icon" type="image/x-icon" href="/images/favicon.ico"> +</head> + + + <body role="document"> + + <nav class="navbar navbar-default navbar-fixed-top"> + <div class="container"> + <div class="navbar-header"> + <a href="/" class="navbar-brand" > + <img alt="Brand" style="height: 25px" src="/images/beam_logo_navbar.png"> + </a> + <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar"> + <span class="sr-only">Toggle navigation</span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + <span class="icon-bar"></span> + </button> + </div> + <div id="navbar" class="navbar-collapse collapse"> + <ul class="nav navbar-nav"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Get Started <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/get-started/beam-overview/">Beam Overview</a></li> + <li><a href="/get-started/quickstart-java/">Quickstart - Java</a></li> + <li><a href="/get-started/quickstart-py/">Quickstart - Python</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Example Walkthroughs</li> + <li><a href="/get-started/wordcount-example/">WordCount</a></li> + <li><a href="/get-started/mobile-gaming-example/">Mobile Gaming</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Resources</li> + <li><a href="/get-started/downloads">Downloads</a></li> + <li><a href="/get-started/support">Support</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Documentation <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/documentation">Using the Documentation</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Beam Concepts</li> + <li><a href="/documentation/programming-guide/">Programming Guide</a></li> + <li><a href="/documentation/resources/">Additional Resources</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Pipeline Fundamentals</li> + <li><a href="/documentation/pipelines/design-your-pipeline/">Design Your Pipeline</a></li> + <li><a href="/documentation/pipelines/create-your-pipeline/">Create Your Pipeline</a></li> + <li><a href="/documentation/pipelines/test-your-pipeline/">Test Your Pipeline</a></li> + <li><a href="/documentation/io/io-toc/">Pipeline I/O</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">SDKs</li> + <li><a href="/documentation/sdks/java/">Java SDK</a></li> + <li><a href="/documentation/sdks/javadoc/0.6.0/" target="_blank">Java SDK API Reference <img src="/images/external-link-icon.png" + width="14" height="14" + alt="External link."></a> + </li> + <li><a href="/documentation/sdks/python/">Python SDK</a></li> + <li><a href="/documentation/sdks/pydoc/0.6.0/" target="_blank">Python SDK API Reference <img src="/images/external-link-icon.png" + width="14" height="14" + alt="External link."></a> + </li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Runners</li> + <li><a href="/documentation/runners/capability-matrix/">Capability Matrix</a></li> + <li><a href="/documentation/runners/direct/">Direct Runner</a></li> + <li><a href="/documentation/runners/apex/">Apache Apex Runner</a></li> + <li><a href="/documentation/runners/flink/">Apache Flink Runner</a></li> + <li><a href="/documentation/runners/spark/">Apache Spark Runner</a></li> + <li><a href="/documentation/runners/dataflow/">Cloud Dataflow Runner</a></li> + </ul> + </li> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">Contribute <span class="caret"></span></a> + <ul class="dropdown-menu"> + <li><a href="/contribute">Get Started Contributing</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Guides</li> + <li><a href="/contribute/contribution-guide/">Contribution Guide</a></li> + <li><a href="/contribute/testing/">Testing Guide</a></li> + <li><a href="/contribute/release-guide/">Release Guide</a></li> + <li><a href="/contribute/ptransform-style-guide/">PTransform Style Guide</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Technical References</li> + <li><a href="/contribute/design-principles/">Design Principles</a></li> + <li><a href="/contribute/work-in-progress/">Ongoing Projects</a></li> + <li><a href="/contribute/source-repository/">Source Repository</a></li> + <li role="separator" class="divider"></li> + <li class="dropdown-header">Promotion</li> + <li><a href="/contribute/presentation-materials/">Presentation Materials</a></li> + <li><a href="/contribute/logos/">Logos and Design</a></li> + <li role="separator" class="divider"></li> + <li><a href="/contribute/maturity-model/">Maturity Model</a></li> + <li><a href="/contribute/team/">Team</a></li> + </ul> + </li> + + <li><a href="/blog">Blog</a></li> + </ul> + <ul class="nav navbar-nav navbar-right"> + <li class="dropdown"> + <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false"><img src="https://www.apache.org/foundation/press/kit/feather_small.png"; alt="Apache Logo" style="height:24px;">Apache Software Foundation<span class="caret"></span></a> + <ul class="dropdown-menu dropdown-menu-right"> + <li><a href="http://www.apache.org/";>ASF Homepage</a></li> + <li><a href="http://www.apache.org/licenses/";>License</a></li> + <li><a href="http://www.apache.org/security/";>Security</a></li> + <li><a href="http://www.apache.org/foundation/thanks.html";>Thanks</a></li> + <li><a href="http://www.apache.org/foundation/sponsorship.html";>Sponsorship</a></li> + <li><a href="https://www.apache.org/foundation/policies/conduct";>Code of Conduct</a></li> + </ul> + </li> + </ul> + </div><!--/.nav-collapse --> + </div> +</nav> + + +<link rel="stylesheet" href=""> + + + <div class="container" role="main"> + + <div class="row"> + <h1 id="using-the-apache-gearpump-runner">Using the Apache Gearpump Runner</h1> + +<p>The Apache Gearpump Runner can be used to execute Beam pipelines using <a href="https://gearpump.apache.org";>Apache Gearpump (incubating)</a>. +When you are running your pipeline with Gearpump Runner you just need to create a jar file containing your job and then it can be executed on a regular Gearpump distributed cluster, or a local cluster which is useful for development and debugging of your pipeline.</p> + +<p>The Gearpump Runner and Gearpump are suitable for large scale, continuous jobs, and provide:</p> + +<ul> + <li>High throughput and low latency stream processing</li> + <li>Comprehensive Dashboard for application monitoring</li> + <li>Fault-tolerance with exactly-once processing guarantees</li> + <li>Application hot re-deployment</li> +</ul> + +<p>The <a href="/documentation/runners/capability-matrix/">Beam Capability Matrix</a> documents the currently supported capabilities of the Gearpump Runner.</p> + +<h2 id="building-gearpump-runner">Building Gearpump Runner</h2> +<p>Currently Gearpump Runner is on a <a href="https://github.com/apache/beam/tree/gearpump-runner";>feature branch</a> and in order to run your Beam pipeline with Gearpump runner, you should build out the corresponding artifacts first.</p> +<div class="highlighter-rouge"><pre class="highlight"><code>git checkout gearpump-runner +mvn install +</code></pre> +</div> + +<h2 id="writing-beam-pipeline-with-gearpump-runner">Writing Beam Pipeline with Gearpump Runner</h2> +<p>To use the Gearpump Runner in a distributed mode, you have to setup a Gearpump cluster first by following the Gearpump <a href="http://gearpump.apache.org/releases/latest/deployment/deployment-standalone/index.html";>setup quickstart</a>.</p> + +<p>Suppose you are writing a Beam pipeline, you can add a dependency on the latest version of the Gearpump runner by adding to your pom.xml to enable Gearpump runner. +And your Beam application should also pack Beam SDK explicitly and here is a snippet of example pom.xml:</p> +<div class="language-java highlighter-rouge"><pre class="highlight"><code><span class="o"><</span><span class="n">dependencies</span><span class="o">></span> + <span class="o"><</span><span class="n">dependency</span><span class="o">></span> + <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">beam</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> + <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">beam</span><span class="o">-</span><span class="n">runners</span><span class="o">-</span><span class="n">gearpump</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> + <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="mf">0.6</span><span class="o">.</span><span class="mi">0</span><span class="o"></</span><span class="n">version</span><span class="o">></span> + <span class="o"></</span><span class="n">dependency</span><span class="o">></span> + + <span class="o"><</span><span class="n">dependency</span><span class="o">></span> + <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">gearpump</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> + <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">gearpump</span><span class="o">-</span><span class="n">streaming_2</span><span class="o">.</span><span class="mi">11</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> + <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="err">$</span><span class="o">{</span><span class="n">gearpump</span><span class="o">.</span><span class="na">version</span><span class="o">}</</span><span class="n">version</span><span class="o">></span> + <span class="o"><</span><span class="n">scope</span><span class="o">></span><span class="n">provided</span><span class="o"></</span><span class="n">scope</span><span class="o">></span> + <span class="o"></</span><span class="n">dependency</span><span class="o">></span> + + <span class="o"><</span><span class="n">dependency</span><span class="o">></span> + <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">gearpump</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> + <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">gearpump</span><span class="o">-</span><span class="n">core_2</span><span class="o">.</span><span class="mi">11</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> + <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="err">$</span><span class="o">{</span><span class="n">gearpump</span><span class="o">.</span><span class="na">version</span><span class="o">}</</span><span class="n">version</span><span class="o">></span> + <span class="o"><</span><span class="n">scope</span><span class="o">></span><span class="n">provided</span><span class="o"></</span><span class="n">scope</span><span class="o">></span> + <span class="o"></</span><span class="n">dependency</span><span class="o">></span> + + <span class="o"><</span><span class="n">dependency</span><span class="o">></span> + <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">beam</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> + <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">beam</span><span class="o">-</span><span class="n">sdks</span><span class="o">-</span><span class="n">java</span><span class="o">-</span><span class="n">core</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> + <span class="o"><</span><span class="n">version</span><span class="o">></span><span class="mf">0.6</span><span class="o">.</span><span class="mi">0</span><span class="o"></</span><span class="n">version</span><span class="o">></span> + <span class="o"></</span><span class="n">dependency</span><span class="o">></span> +<span class="o"></</span><span class="n">dependencies</span><span class="o">></span> + +<span class="o"><</span><span class="n">build</span><span class="o">></span> + <span class="o"><</span><span class="n">plugins</span><span class="o">></span> + <span class="o"><</span><span class="n">plugin</span><span class="o">></span> + <span class="o"><</span><span class="n">groupId</span><span class="o">></span><span class="n">org</span><span class="o">.</span><span class="na">apache</span><span class="o">.</span><span class="na">maven</span><span class="o">.</span><span class="na">plugins</span><span class="o"></</span><span class="n">groupId</span><span class="o">></span> + <span class="o"><</span><span class="n">artifactId</span><span class="o">></span><span class="n">maven</span><span class="o">-</span><span class="n">shade</span><span class="o">-</span><span class="n">plugin</span><span class="o"></</span><span class="n">artifactId</span><span class="o">></span> + <span class="o"><</span><span class="n">configuration</span><span class="o">></span> + <span class="o"><</span><span class="n">createDependencyReducedPom</span><span class="o">></span><span class="kc">false</span><span class="o"></</span><span class="n">createDependencyReducedPom</span><span class="o">></span> + <span class="o"><</span><span class="n">filters</span><span class="o">></span> + <span class="o"><</span><span class="n">filter</span><span class="o">></span> + <span class="o"><</span><span class="n">artifact</span><span class="o">>*:*</</span><span class="n">artifact</span><span class="o">></span> + <span class="o"><</span><span class="n">excludes</span><span class="o">></span> + <span class="o"><</span><span class="n">exclude</span><span class="o">></span><span class="n">META</span><span class="o">-</span><span class="n">INF</span><span class="o">/*.</span><span class="na">SF</span><span class="o"></</span><span class="n">exclude</span><span class="o">></span> + <span class="o"><</span><span class="n">exclude</span><span class="o">></span><span class="n">META</span><span class="o">-</span><span class="n">INF</span><span class="o">/*.</span><span class="na">DSA</span><span class="o"></</span><span class="n">exclude</span><span class="o">></span> + <span class="o"><</span><span class="n">exclude</span><span class="o">></span><span class="n">META</span><span class="o">-</span><span class="n">INF</span><span class="o">/*.</span><span class="na">RSA</span><span class="o"></</span><span class="n">exclude</span><span class="o">></span> + <span class="o"></</span><span class="n">excludes</span><span class="o">></span> + <span class="o"></</span><span class="n">filter</span><span class="o">></span> + <span class="o"></</span><span class="n">filters</span><span class="o">></span> + <span class="o"></</span><span class="n">configuration</span><span class="o">></span> + <span class="o"><</span><span class="n">executions</span><span class="o">></span> + <span class="o"><</span><span class="n">execution</span><span class="o">></span> + <span class="o"><</span><span class="n">phase</span><span class="o">></span><span class="kn">package</span><span class="o"></</span><span class="n">phase</span><span class="o">></span> + <span class="o"><</span><span class="n">goals</span><span class="o">></span> + <span class="o"><</span><span class="n">goal</span><span class="o">></span><span class="n">shade</span><span class="o"></</span><span class="n">goal</span><span class="o">></span> + <span class="o"></</span><span class="n">goals</span><span class="o">></span> + <span class="o"><</span><span class="n">configuration</span><span class="o">></span> + <span class="o"><</span><span class="n">shadedArtifactAttached</span><span class="o">></span><span class="kc">true</span><span class="o"></</span><span class="n">shadedArtifactAttached</span><span class="o">></span> + <span class="o"><</span><span class="n">shadedClassifierName</span><span class="o">></span><span class="n">shaded</span><span class="o"></</span><span class="n">shadedClassifierName</span><span class="o">></span> + <span class="o"></</span><span class="n">configuration</span><span class="o">></span> + <span class="o"></</span><span class="n">execution</span><span class="o">></span> + <span class="o"></</span><span class="n">executions</span><span class="o">></span> + <span class="o"></</span><span class="n">plugin</span><span class="o">></span> + <span class="o"></</span><span class="n">plugins</span><span class="o">></span> +<span class="o"></</span><span class="n">build</span><span class="o">></span> +</code></pre> +</div> + +<p>After running <code>mvn package</code>, run <code>ls target</code> and you should see your application jar like:</p> +<div class="highlighter-rouge"><pre class="highlight"><code><span class="p">{</span><span class="err">your_application</span><span class="p">}</span><span class="err">-</span><span class="p">{</span><span class="err">version</span><span class="p">}</span><span class="err">-shaded.jar</span><span class="w"> +</span></code></pre> +</div> + +<h2 id="executing-the-pipeline-on-a-gearpump-cluster">Executing the pipeline on a Gearpump cluster</h2> +<p>To run against a Gearpump cluster simply run:</p> +<div class="highlighter-rouge"><pre class="highlight"><code>gear app -jar /path/to/{your_application}-{version}-shaded.jar com.beam.examples.BeamPipeline --runner=GearpumpRunner ... +</code></pre> +</div> + +<h2 id="monitoring-your-application">Monitoring your application</h2> +<p>You can monitor a running Gearpump application using Gearpumpâs Dashboard. Please follow the Gearpump <a href="http://gearpump.apache.org/releases/latest/deployment/deployment-standalone/index.html#start-ui";>Start UI</a> to start the dashboard.</p> + +<h2 id="pipeline-options-for-the-gearpump-runner">Pipeline options for the Gearpump Runner</h2> + +<p>When executing your pipeline with the Gearpump Runner, you should consider the following pipeline options.</p> + +<table class="table table-bordered"> +<tr> + <th>Field</th> + <th>Description</th> + <th>Default Value</th> +</tr> +<tr> + <td><code>runner</code></td> + <td>The pipeline runner to use. This option allows you to determine the pipeline runner at runtime.</td> + <td>Set to <code>GearpumpRunner</code> to run using Gearpump.</td> +</tr> +<tr> + <td><code>parallelism</code></td> + <td>The parallelism for Gearpump processor.</td> + <td><code>1</code></td> +</tr> +<tr> + <td><code>applicationName</code></td> + <td>The application name for Gearpump runner.</td> + <td><code>beam_gearpump_app</code></td> +</tr> +</table> + + + </div> + + + <hr> + <div class="row"> + <div class="col-xs-12"> + <footer> + <p class="text-center"> + © Copyright + <a href="http://www.apache.org";>The Apache Software Foundation</a>, + 2017. All Rights Reserved. + </p> + <p class="text-center"> + <a href="/privacy_policy">Privacy Policy</a> | + <a href="/feed.xml">RSS Feed</a> + </p> + </footer> + </div> + </div> + <!-- container div end --> +</div> + + + </body> + +</html> http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/get-started/beam-overview/index.html ---------------------------------------------------------------------- diff --git a/content/get-started/beam-overview/index.html b/content/get-started/beam-overview/index.html index c284b15..6bae7d3 100644 --- a/content/get-started/beam-overview/index.html +++ b/content/get-started/beam-overview/index.html @@ -177,11 +177,21 @@ <ul> <li>Apache Apex <img src="/images/logos/runners/apex.png" alt="Apache Apex" /></li> <li>Apache Flink <img src="/images/logos/runners/flink.png" alt="Apache Flink" /></li> + <li>Apache Gearpump (incubating) <img src="/images/logos/runners/gearpump.png" alt="Apache Gearpump" /></li> <li>Apache Spark <img src="/images/logos/runners/spark.png" alt="Apache Spark" /></li> <li>Google Cloud Dataflow <img src="/images/logos/runners/dataflow.png" alt="Google Cloud Dataflow" /></li> </ul> -<p><strong>Note:</strong> You can always execute your pipeline locally for testing and debugging purposes.</p> +<p><strong>Note:</strong></p> + +<ol> + <li> + <p>You can always execute your pipeline locally for testing and debugging purposes.</p> + </li> + <li> + <p>Gearpump Runner is under development and for more details please refer to <a href="/contribute/work-in-progress">Ongoing Projects</a></p> + </li> +</ol> <h2 id="get-started">Get Started</h2> http://git-wip-us.apache.org/repos/asf/beam-site/blob/d7f46849/content/images/logos/runners/gearpump.png ---------------------------------------------------------------------- diff --git a/content/images/logos/runners/gearpump.png b/content/images/logos/runners/gearpump.png new file mode 100644 index 0000000..d35f44a Binary files /dev/null and b/content/images/logos/runners/gearpump.png differ
![https://www.apache.org/foundation/press/kit/feather_small.png"](https://www.apache.org/foundation/press/kit/feather_small.png"){kind=link}
