Added: samza/site/startup/code-examples/versioned/samza.html URL: http://svn.apache.org/viewvc/samza/site/startup/code-examples/versioned/samza.html?rev=1855804&view=auto ============================================================================== --- samza/site/startup/code-examples/versioned/samza.html (added) +++ samza/site/startup/code-examples/versioned/samza.html Tue Mar 19 05:31:11 2019 @@ -0,0 +1,664 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + +<head> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> + <meta charset="utf-8"> + <title>Samza - Samza Code Examples</title> + <link rel="apple-touch-icon-precomposed" sizes="57x57" href="/img/favicon/apple-touch-icon-57x57.png" /> + <link rel="apple-touch-icon-precomposed" sizes="114x114" href="/img/favicon/apple-touch-icon-114x114.png" /> + <link rel="apple-touch-icon-precomposed" sizes="72x72" href="/img/favicon/apple-touch-icon-72x72.png" /> + <link rel="apple-touch-icon-precomposed" sizes="144x144" href="/img/favicon/apple-touch-icon-144x144.png" /> + <link rel="apple-touch-icon-precomposed" sizes="60x60" href="/img/favicon/apple-touch-icon-60x60.png" /> + <link rel="apple-touch-icon-precomposed" sizes="120x120" href="/img/favicon/apple-touch-icon-120x120.png" /> + <link rel="apple-touch-icon-precomposed" sizes="76x76" href="/img/favicon/apple-touch-icon-76x76.png" /> + <link rel="apple-touch-icon-precomposed" sizes="152x152" href="/img/favicon/apple-touch-icon-152x152.png" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-196x196.png" sizes="196x196" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-96x96.png" sizes="96x96" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-16x16.png" sizes="16x16" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-128.png" sizes="128x128" /> + <meta name="application-name" content="https://samza.apache.org"; /> + <meta name="msapplication-TileColor" content="#FFFFFF" /> + <meta name="msapplication-TileImage" content="/img/favicon/mstile-144x144.png" /> + <meta name="msapplication-square70x70logo" content="/img/favicon/mstile-70x70.png" /> + <meta name="msapplication-square150x150logo" content="/img/favicon/mstile-150x150.png" /> + <meta name="msapplication-wide310x150logo" content="/img/favicon/mstile-310x150.png" /> + <meta name="msapplication-square310x310logo" content="/img/favicon/mstile-310x310.png" /> + <link href="/css/ionicons.min.css" rel="stylesheet"> + <link href="/css/google-fonts.css" rel="stylesheet"> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="stylesheet" href="/css/main.new.css" /> +</head> + +<body class="page"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<div class="main-navigation" data-plugin="menu"> + <div class="main-navigation__toggle" data-menu-closed> + <i class="icon ion-md-menu"></i> + </div> + <div class="main-navigation__toggle main-navigation__toggle--opened" data-menu-opened> + <i class="icon ion-md-close"></i> + </div> + <div class="main-navigation__inner"> + <div class="main-navigation__logo"> + <a href="/"> + <img class="main-navigation__logo-img" src="/img/samza-logo.png" srcset="/img/samza-logo.png 1x, /img/[email protected] 2x" + alt="Samza Logo" /> + </a> + </div> + <div class="main-navigation__items" data-menu-opened> + <a class="main-navigation__item" href="/">Home</a> + <a class="main-navigation__item" href="/learn/documentation/latest/core-concepts/core-concepts.html">Docs</a> + <a class="main-navigation__item" href="/powered-by/">Powered By</a> + <a class="main-navigation__item" href="/startup/download/">Downloads</a> + <a class="main-navigation__item" href="/blog/">Blog</a> + <div class="main-navigation__item main-navigation__item--group"> + <div class="main-navigation__item-group-title"> + Community + <i class="icon ion-md-arrow-dropdown"></i> + </div> + <div class="main-navigation__item-group-list"> + <a class="main-navigation__item" href="/community/contact-us.html">Contact Us</a> + <a class="main-navigation__item" href="/contribute/contributors-corner.html">Contributor's Corner</a> + <a class="main-navigation__item" href="/community/committers.html">PMC Members and committers</a> + <a class="main-navigation__item" href="/meetups/">Talks and Meetups</a> + </div> + </div> + </div> + </div> +</div> + + <div class="container"> + <div class="container__toggle"> + <i class="icon ion-md-arrow-dropleft-circle container__toggle-icon"></i> + <i class="icon ion-md-arrow-dropright-circle container__toggle-icon container__toggle-icon--opened"></i> + </div> + + <!-- There is only one menu, but made it as a no-output collection to grab data only --> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + +<div class="side-navigation"> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Getting Started + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/latest/">QuickStart</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/latest/">Code Examples</a> + + + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Documentation + </div> + <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/latest/"> + + + <!-- Handle sub navigation items from data --> + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Releases + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/1.0.0">1.0.0</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.14">0.14</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.13">0.13</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.12">0.12</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.11">0.11</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.10">0.10</a> + + + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group"> + + + <!-- Make menu_title, and start items group if needed --> + + <a class="side-navigation__group-title" data-plugin="top-menu" data-match-active="" href="/blog/"> + Blog + </a> + + + <!-- Handle sub navigation items from data --> + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Community + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="" href="/community/contact-us.html">Contact Us</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/contribute/contributors-corner.html">Contributor's Corner</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/contribute/enhancement-proposal.html">Enhancement Proposal</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/community/committers.html">PMC members & Committers</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/meetups/">Talks and Meetups</a> + + + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Case Studies + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="exact" href="/case-studies/">View All</a> + + + <hr> + + + + + <!-- Handle sub nagivation from site collections --> + + + + + + + + + + + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/ebay" data-match-active="">eBay</a> + + + + + + + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/tripadvisor" data-match-active="">TripAdvisor</a> + + + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/slack" data-match-active="">Slack</a> + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/optimizely" data-match-active="">Optimizely</a> + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/redfin" data-match-active="">Redfin</a> + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/linkedin" data-match-active="">LinkedIn</a> + + + + + + + + + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + +</div> + + + + <div class="section"> + <div class="content"> + + <h2>Samza Code Examples</h2> + + + + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h3 id="checking-out-our-examples">Checking out our examples</h3> + +<p>The <a href="https://github.com/apache/samza-hello-samza";>hello-samza</a> project contains several examples to help you create your Samza applications. To checkout the hello-samza project:</p> + +<figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>> git clone https://git.apache.org/samza-hello-samza.git hello-samza</code></pre></figure> + +<h4 id="high-level-api-examples">High-level API examples</h4> + +<p><a href="https://github.com/apache/samza-hello-samza/tree/master/src/main/java/samza/examples/cookbook";>The Samza Cookbook</a> contains various recipes using the Samza high-level API. +These include:</p> + +<ul> +<li><p>The <a href="https://github.com/apache/samza-hello-samza/blob/latest/src/main/java/samza/examples/cookbook/FilterExample.java";>Filter example</a> demonstrates how to perform stateless operations on a stream. </p></li> +<li><p>The <a href="https://github.com/apache/samza-hello-samza/blob/latest/src/main/java/samza/examples/cookbook/JoinExample.java";>Join example</a> demonstrates how you can join a Kafka stream of page-views with a stream of ad-clicks</p></li> +<li><p>The <a href="https://github.com/apache/samza-hello-samza/blob/latest/src/main/java/samza/examples/cookbook/RemoteTableJoinExample.java";>Stream-Table Join example</a> demonstrates how to use the Samza Table API. It joins a Kafka stream with a remote dataset accessed through a REST service.</p></li> +<li><p>The <a href="https://github.com/apache/samza-hello-samza/blob/latest/src/main/java/samza/examples/cookbook/SessionWindowExample.java";>SessionWindow</a> and <a href="https://github.com/apache/samza-hello-samza/blob/latest/src/main/java/samza/examples/cookbook/TumblingWindowExample.java";>TumblingWindow</a> examples illustrate Samza’s rich windowing and triggering capabilities.</p></li> +</ul> + +<p>In addition to the cookbook, you can also consult these:</p> + +<ul> +<li><p><a href="https://github.com/apache/samza-hello-samza/tree/master/src/main/java/samza/examples/wikipedia";>Wikipedia Parser</a>: An advanced example that builds a streaming pipeline consuming a live-feed of wikipedia edits, parsing each message and generating statistics from them.</p></li> +<li><p><a href="https://github.com/apache/samza-hello-samza/tree/master/src/main/java/samza/examples/kinesis";>Amazon Kinesis</a> and <a href="https://github.com/apache/samza-hello-samza/tree/latest/src/main/java/samza/examples/azure";>Azure Eventhubs</a> examples that cover how to consume input data from the respective systems.</p></li> +</ul> + +<h4 id="low-level-api-examples">Low-level API examples</h4> + +<p>The <a href="https://github.com/apache/samza-hello-samza/tree/latest/src/main/java/samza/examples/wikipedia/task/application";>Wikipedia Parser (low-level API)</a>: +Same example that builds a streaming pipeline consuming a live-feed of +wikipedia edits, parsing each message and generating statistics from them, but +using low-level APIs. </p> + +<h4 id="samza-sql-api-examples">Samza SQL API examples</h4> + +<p>You can easily create a Samza job declaratively using +<a href="https://samza.apache.org/learn/tutorials/latest/samza-sql.html";>Samza SQL</a>.</p> + + + </div> + </div> + + </div> + + + <!-- footer starts here --> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<footer> + <div class="footer-inner"> + <div class="side-by-side"> + <div> + <div class="footer__heading">Learn More</div> + <div class="footer__items"> + <a class="footer__item" href="/meetups/">Meetups</a> + <a class="footer__item" href="/blog/">Blog</a> + <a class="footer__item" href="/learn/documentation/latest/introduction/background.html">About</a> + </div> + </div> + <div> + <div class="footer__heading">Community</div> + <div class="footer__items"> + <a class="footer__item" href="/community/contact-us.html">Contact Us</a> + <a class="footer__item" href="/contribute/contributors-corner.html">Contributors' Corner</a> + <a class="footer__item" href="/community/committers.html">PMC members and committers</a> + <a class="footer__item" href="/powered-by/">Powered By</a> + </div> + </div> + + <div> + <div class="quick-links"> + <a class="quick-link" href="/startup/download" target="_blank"> + <i class="icon ion-md-download"></i> + </a> + <a class="quick-link" href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"> + <i class="icon ion-md-code"></i> + </a> + <a class="quick-link" href="https://twitter.com/samzastream"; target="_blank"> + <i class="icon ion-logo-twitter"></i> + </a> + </div> + + <p> + <script>document.write(new Date().getFullYear());</script> © samza.apache.org</p> + </div> + + </div> + </div> + +</footer> + + +<script> + var tryFile = function (url, cb) { + var myRequest = new Request(url); + fetch(myRequest).then((response, cb) => { + console.log(response.status); // returns 200 + cb(response.status != 404); + }); + } + + tryFile(window.location.pathname, function (status) { + // do something with the status + console.log(status); + }); +</script> + + +<!-- Google Analytics --> +<script> + (function (i, s, o, g, r, a, m) { + i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function () { + (i[r].q = i[r].q || []).push(arguments) + }, i[r].l = 1 * new Date(); a = s.createElement(o), + m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g; m.parentNode.insertBefore(a, m) + })(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + +</script> +<script src="/js/main.new.js"></script> + +</body> + +</html> \ No newline at end of file
Modified: samza/site/startup/download/index.html URL: http://svn.apache.org/viewvc/samza/site/startup/download/index.html?rev=1855804&r1=1855803&r2=1855804&view=diff ============================================================================== --- samza/site/startup/download/index.html (original) +++ samza/site/startup/download/index.html Tue Mar 19 05:31:11 2019 @@ -81,7 +81,7 @@ </div> <div class="main-navigation__items" data-menu-opened> <a class="main-navigation__item" href="/">Home</a> - <a class="main-navigation__item" href="/learn/documentation/1.0.0/core-concepts/core-concepts.html">Docs</a> + <a class="main-navigation__item" href="/learn/documentation/latest/core-concepts/core-concepts.html">Docs</a> <a class="main-navigation__item" href="/powered-by/">Powered By</a> <a class="main-navigation__item" href="/startup/download/">Downloads</a> <a class="main-navigation__item" href="/blog/">Blog</a> @@ -152,10 +152,10 @@ - <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/1.0.0/">QuickStart</a> + <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/latest/">QuickStart</a> - <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/1.0.0/">Code Examples</a> + <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/latest/">Code Examples</a> @@ -187,7 +187,7 @@ <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Documentation </div> - <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/1.0.0/"> + <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/latest/"> <!-- Handle sub navigation items from data --> @@ -523,7 +523,7 @@ <p>Samza is released as a source artifact, and also through Maven.</p> -<p>If you just want to play around with Samza for the first time, go to <a href="/startup/hello-samza/1.0.0">Hello Samza</a>.</p> +<p>If you just want to play around with Samza for the first time, go to <a href="/startup/hello-samza/latest">Hello Samza</a>.</p> <h3 id="jdk-notice">JDK Notice</h3> @@ -628,7 +628,7 @@ <span class="nt"><version></span>0.8.1<span class="nt"></version></span> <span class="nt"></dependency></span></code></pre></figure> -<p><a href="/startup/hello-samza/1.0.0">Hello Samza</a> is a working Maven project that illustrates how to build projects that have Samza jobs in them.</p> +<p><a href="/startup/hello-samza/latest">Hello Samza</a> is a working Maven project that illustrates how to build projects that have Samza jobs in them.</p> <h4 id="repositories">Repositories</h4> @@ -690,7 +690,7 @@ <div class="footer__items"> <a class="footer__item" href="/meetups/">Meetups</a> <a class="footer__item" href="/blog/">Blog</a> - <a class="footer__item" href="/learn/documentation/1.0.0/introduction/background.html">About</a> + <a class="footer__item" href="/learn/documentation/latest/introduction/background.html">About</a> </div> </div> <div> Modified: samza/site/startup/preview/index.html URL: http://svn.apache.org/viewvc/samza/site/startup/preview/index.html?rev=1855804&r1=1855803&r2=1855804&view=diff ============================================================================== --- samza/site/startup/preview/index.html (original) +++ samza/site/startup/preview/index.html Tue Mar 19 05:31:11 2019 @@ -81,7 +81,7 @@ </div> <div class="main-navigation__items" data-menu-opened> <a class="main-navigation__item" href="/">Home</a> - <a class="main-navigation__item" href="/learn/documentation/1.0.0/core-concepts/core-concepts.html">Docs</a> + <a class="main-navigation__item" href="/learn/documentation/latest/core-concepts/core-concepts.html">Docs</a> <a class="main-navigation__item" href="/powered-by/">Powered By</a> <a class="main-navigation__item" href="/startup/download/">Downloads</a> <a class="main-navigation__item" href="/blog/">Blog</a> @@ -152,10 +152,10 @@ - <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/1.0.0/">QuickStart</a> + <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/latest/">QuickStart</a> - <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/1.0.0/">Code Examples</a> + <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/latest/">Code Examples</a> @@ -187,7 +187,7 @@ <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Documentation </div> - <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/1.0.0/"> + <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/latest/"> <!-- Handle sub navigation items from data --> @@ -542,9 +542,9 @@ <p>Want to skip all the details and get some hands on experience? There are three tutorials to help you get acquainted with running Samza applications in both YARN and embedded modes and programming with the high level API:</p> <ul> -<li><a href="/learn/tutorials/1.0.0/hello-samza-high-level-yarn.html">Yarn Deployment</a> - run a pre-existing Wikipedia application on YARN and observe the output.</li> -<li><a href="/learn/tutorials/1.0.0/hello-samza-high-level-code.html">High Level API Code Walkthrough</a> - walk through building the Wikipedia application, step by step.</li> -<li><a href="/learn/tutorials/1.0.0/hello-samza-high-level-zk.html">ZooKeeper Deployment</a> - run a pre-existing Wikipedia application with ZooKeeper coordination and observe the output.</li> +<li><a href="/learn/tutorials/latest/hello-samza-high-level-yarn.html">Yarn Deployment</a> - run a pre-existing Wikipedia application on YARN and observe the output.</li> +<li><a href="/learn/tutorials/latest/hello-samza-high-level-code.html">High Level API Code Walkthrough</a> - walk through building the Wikipedia application, step by step.</li> +<li><a href="/learn/tutorials/latest/hello-samza-high-level-zk.html">ZooKeeper Deployment</a> - run a pre-existing Wikipedia application with ZooKeeper coordination and observe the output.</li> </ul> <hr> @@ -559,19 +559,19 @@ <p>The Samza architecture has been overhauled with distinct layers to handle each stage of application development. The following diagram shows an overview of Apache Samza architecture with the high level API.</p> -<p><img src="/img/1.0.0/learn/documentation/introduction/layered-arch.png" alt="Architecture diagram" style="max-width: 100%; height: auto;" onclick="window.open(this.src)"></p> +<p><img src="/img/latest/learn/documentation/introduction/layered-arch.png" alt="Architecture diagram" style="max-width: 100%; height: auto;" onclick="window.open(this.src)"></p> <p>There are four layers in the architecture. The following sections describe each of the layers.</p> <h4 id="i-high-level-api">I. High Level API</h4> -<p>The high level API provides the libraries to define your application logic. The <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/application/StreamApplication.html">StreamApplication</a> is the central abstraction which your application must implement. You start by declaring your inputs as instances of <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/MessageStream.html">MessageStream</a>. Then you can apply operators on each MessageStream like map, filter, window, and join to define the whole end-to-end data processing in a single program.</p> +<p>The high level API provides the libraries to define your application logic. The <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/application/StreamApplication.html">StreamApplication</a> is the central abstraction which your application must implement. You start by declaring your inputs as instances of <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/MessageStream.html">MessageStream</a>. Then you can apply operators on each MessageStream like map, filter, window, and join to define the whole end-to-end data processing in a single program.</p> <p>For a deeper dive into the high level API, see <a href="#high-level-api">high level API section</a> below.</p> <h4 id="ii-applicationrunner">II. ApplicationRunner</h4> -<p>Samza uses an <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/runtime/ApplicationRunner.html">ApplicationRunner</a> to run a stream application. The ApplicationRunner generates the configs (such as input/output streams), creates intermediate streams, and starts the execution. There are two types of ApplicationRunner:</p> +<p>Samza uses an <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/runtime/ApplicationRunner.html">ApplicationRunner</a> to run a stream application. The ApplicationRunner generates the configs (such as input/output streams), creates intermediate streams, and starts the execution. There are two types of ApplicationRunner:</p> <p><strong>RemoteApplicationRunner</strong> - submits the application to a remote cluster. This runner is invoked via the <em>run-app.sh</em> script. To use RemoteApplicationRunner, set the following configurations</p> @@ -579,7 +579,7 @@ <span class="na">app.class</span><span class="o">=</span><span class="s">com.company.job.YourStreamApplication</span> <span class="na">job.factory.class</span><span class="o">=</span><span class="s">org.apache.samza.job.yarn.YarnJobFactory</span></code></pre></figure> -<p>Then use <em>run-app.sh</em> to run the application in the remote cluster. The script will invoke the RemoteApplicationRunner, which will launch one or more jobs using the factory specified with <em>job.factory.class</em>. Follow the <a href="/learn/tutorials/1.0.0/hello-samza-high-level-yarn.html">yarn deployment tutorial</a> to try it out.</p> +<p>Then use <em>run-app.sh</em> to run the application in the remote cluster. The script will invoke the RemoteApplicationRunner, which will launch one or more jobs using the factory specified with <em>job.factory.class</em>. Follow the <a href="/learn/tutorials/latest/hello-samza-high-level-yarn.html">yarn deployment tutorial</a> to try it out.</p> <p><strong>LocalApplicationRunner</strong> - runs the application in the JVM process of the runner. For example, to launch your application on multiple machines using ZooKeeper for coordination, you can run multiple instances of LocalApplicationRunner on various machines. After the applications load they will start cordinatinating their actions through ZooKeeper. Here is an example to run the StreamApplication in your program using the LocalApplicationRunner:</p> @@ -595,7 +595,7 @@ <span class="n">System</span><span class="o">.</span><span class="na">out</span><span class="o">.</span><span class="na">println</span><span class="o">(</span><span class="s">"Application completed with status "</span> <span class="o">+</span> <span class="n">localRunner</span><span class="o">.</span><span class="na">status</span><span class="o">(</span><span class="n">app</span><span class="o">));</span> <span class="o">}</span></code></pre></figure> -<p>Follow the <a href="/learn/tutorials/1.0.0/hello-samza-high-level-zk.html">ZooKeeper deployment tutorial</a> to try it out.</p> +<p>Follow the <a href="/learn/tutorials/latest/hello-samza-high-level-zk.html">ZooKeeper deployment tutorial</a> to try it out.</p> <h5 id="execution-plan">Execution Plan</h5> @@ -608,7 +608,7 @@ <p>To view the plan, open the <em>bin/plan.html</em> file in a browser. Here’s a sample plan visualization:</p> -<p><img src="/img/1.0.0/learn/documentation/introduction/execution-plan.png" alt="Execution plan" style="max-width: 100%; height: auto;" onclick="window.open(this.src)"/></p> +<p><img src="/img/latest/learn/documentation/introduction/execution-plan.png" alt="Execution plan" style="max-width: 100%; height: auto;" onclick="window.open(this.src)"/></p> <h4 id="iii-execution-models">III. Execution Models</h4> @@ -627,7 +627,7 @@ <h4 id="iv-processor">IV. Processor</h4> -<p>The lowest execution unit of a Samza application is the processor. It reads the configs generated from the <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/runtime/ApplicationRunner.html">ApplicationRunner</a> and processes the input stream partitions assigned by the JobCoordinator. It can access local state using a <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html">KeyValueStore</a> implementation (e.g. RocksDB or in-memory) and remote state (e.g. REST service) using multithreading.</p> +<p>The lowest execution unit of a Samza application is the processor. It reads the configs generated from the <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/runtime/ApplicationRunner.html">ApplicationRunner</a> and processes the input stream partitions assigned by the JobCoordinator. It can access local state using a <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/storage/kv/KeyValueStore.html">KeyValueStore</a> implementation (e.g. RocksDB or in-memory) and remote state (e.g. REST service) using multithreading.</p> <hr> @@ -650,7 +650,7 @@ <h3 id="streamapplication">StreamApplication</h3> -<p>When writing your stream processing application using the Samza high-level API, you should implement a <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/application/StreamApplication.html">StreamApplication</a> and define your processing logic in the init method.</p> +<p>When writing your stream processing application using the Samza high-level API, you should implement a <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/application/StreamApplication.html">StreamApplication</a> and define your processing logic in the init method.</p> <figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="kd">public</span> <span class="kt">void</span> <span class="nf">init</span><span class="o">(</span><span class="n">StreamGraph</span> <span class="n">graph</span><span class="o">,</span> <span class="n">Config</span> <span class="n">config</span><span class="o">)</span> <span class="o">{</span> <span class="err">â¦</span> <span class="o">}</span></code></pre></figure> @@ -723,11 +723,11 @@ <h2 id="operators">Operators</h2> -<p>The high level API supports common operators like map, flatmap, filter, merge, joins, and windowing on streams. Most of these operators accept corresponding Functions and these functions are <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/InitableFunction.html">Initable</a>.</p> +<p>The high level API supports common operators like map, flatmap, filter, merge, joins, and windowing on streams. Most of these operators accept corresponding Functions and these functions are <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/InitableFunction.html">Initable</a>.</p> <h3 id="map">Map</h3> -<p>Applies the provided 1:1 <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/MapFunction.html">MapFunction</a> to each element in the MessageStream and returns the transformed MessageStream. The MapFunction takes in a single message and returns a single message (potentially of a different type).</p> +<p>Applies the provided 1:1 <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/MapFunction.html">MapFunction</a> to each element in the MessageStream and returns the transformed MessageStream. The MapFunction takes in a single message and returns a single message (potentially of a different type).</p> <figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="n">MessageStream</span><span class="o"><</span><span class="n">Integer</span><span class="o">></span> <span class="n">numbers</span> <span class="o">=</span> <span class="o">...</span> <span class="n">MessageStream</span><span class="o"><</span><span class="n">Integer</span><span class="o">></span> <span class="n">tripled</span><span class="o">=</span> <span class="n">numbers</span><span class="o">.</span><span class="na">map</span><span class="o">(</span><span class="n">m</span> <span class="o">-></span> <span class="n">m</span> <span class="o">*</span> <span class="mi">3</span><span class="o">)</span> @@ -735,7 +735,7 @@ <h3 id="flatmap">Flatmap</h3> -<p>Applies the provided 1:n <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/FlatMapFunction.html">FlatMapFunction</a> to each element in the MessageStream and returns the transformed MessageStream. The FlatMapFunction takes in a single message and returns zero or more messages.</p> +<p>Applies the provided 1:n <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/FlatMapFunction.html">FlatMapFunction</a> to each element in the MessageStream and returns the transformed MessageStream. The FlatMapFunction takes in a single message and returns zero or more messages.</p> <figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="n">MessageStream</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">sentence</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// Parse the sentence into its individual words splitting by space</span> @@ -744,7 +744,7 @@ <h3 id="filter">Filter</h3> -<p>Applies the provided <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/FilterFunction.html">FilterFunction</a> to the MessageStream and returns the filtered MessageStream. The FilterFunction is a predicate that specifies whether a message should be retained in the filtered stream. Messages for which the FilterFunction returns false are filtered out.</p> +<p>Applies the provided <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/FilterFunction.html">FilterFunction</a> to the MessageStream and returns the filtered MessageStream. The FilterFunction is a predicate that specifies whether a message should be retained in the filtered stream. Messages for which the FilterFunction returns false are filtered out.</p> <figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="n">MessageStream</span><span class="o"><</span><span class="n">String</span><span class="o">></span> <span class="n">words</span> <span class="o">=</span> <span class="o">...</span> <span class="c1">// Extract only the long words</span> @@ -777,7 +777,7 @@ <p>The merge transform preserves the order of each MessageStream, so if message <code>m1</code> appears before <code>m2</code> in any provided stream, then, <code>m1</code> also appears before <code>m2</code> in the merged stream.</p> -<p>As an alternative to the <code>merge</code> instance method, you also can use the <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/MessageStream.html#mergeAll-java.util.Collection-">MessageStream#mergeAll</a> static method to merge MessageStreams without operating on an initial stream.</p> +<p>As an alternative to the <code>merge</code> instance method, you also can use the <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/MessageStream.html#mergeAll-java.util.Collection-">MessageStream#mergeAll</a> static method to merge MessageStreams without operating on an initial stream.</p> <h3 id="sendto-stream">SendTo (stream)</h3> @@ -802,7 +802,7 @@ <h3 id="sink">Sink</h3> -<p>Allows sending messages from this MessageStream to an output system using the provided <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/SinkFunction.html">SinkFunction</a>.</p> +<p>Allows sending messages from this MessageStream to an output system using the provided <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/SinkFunction.html">SinkFunction</a>.</p> <p>This offers more control than sendTo since the SinkFunction has access to the <code>MessageCollector</code> and the <code>TaskCoordinator</code>. For instance, you can choose to manually commit offsets, or shut-down the job using the TaskCoordinator APIs. This operator can also be used to send messages to non-Samza systems (e.g. remote databases, REST services, etc.)</p> @@ -816,7 +816,7 @@ <h3 id="join-stream-stream">Join (stream-stream)</h3> -<p>The stream-stream Join operator joins messages from two MessageStreams using the provided pairwise <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/JoinFunction.html">JoinFunction</a>. Messages are joined when the keys extracted from messages from the first stream match keys extracted from messages in the second stream. Messages in each stream are retained for the provided ttl duration and join results are emitted as matches are found.</p> +<p>The stream-stream Join operator joins messages from two MessageStreams using the provided pairwise <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/JoinFunction.html">JoinFunction</a>. Messages are joined when the keys extracted from messages from the first stream match keys extracted from messages in the second stream. Messages in each stream are retained for the provided ttl duration and join results are emitted as matches are found.</p> <figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="c1">// Joins a stream of OrderRecord with a stream of ShipmentRecord by orderId with a TTL of 20 minutes.</span> <span class="c1">// Results are produced to a new stream of FulfilledOrderRecord.</span> @@ -849,7 +849,7 @@ <h3 id="join-stream-table">Join (stream-table)</h3> -<p>The stream-table Join operator joins messages from a MessageStream using the provided <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/StreamTableJoinFunction.html">StreamTableJoinFunction</a>. Messages from the input stream are joined with record in table using key extracted from input messages. The join function is invoked with both the message and the record. If a record is not found in the table, a null value is provided; the join function can choose to return null (inner join) or an output message (left outer join). For join to function properly, it is important to ensure the input stream and table are partitioned using the same key as this impacts the physical placement of data. </p> +<p>The stream-table Join operator joins messages from a MessageStream using the provided <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/StreamTableJoinFunction.html">StreamTableJoinFunction</a>. Messages from the input stream are joined with record in table using key extracted from input messages. The join function is invoked with both the message and the record. If a record is not found in the table, a null value is provided; the join function can choose to return null (inner join) or an output message (left outer join). For join to function properly, it is important to ensure the input stream and table are partitioned using the same key as this impacts the physical placement of data. </p> <figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span> <span class="n">streamGraph</span><span class="o">.</span><span class="na">getInputStream</span><span class="o">(</span><span class="s">"PageView"</span><span class="o">,</span> <span class="k">new</span> <span class="n">NoOpSerde</span><span class="o"><</span><span class="n">PageView</span><span class="o">>())</span> <span class="o">.</span><span class="na">partitionBy</span><span class="o">(</span><span class="n">PageView</span><span class="o">::</span><span class="n">getMemberId</span><span class="o">,</span> <span class="n">v</span> <span class="o">-></span> <span class="n">v</span><span class="o">,</span> <span class="s">"p1"</span><span class="o">)</span> @@ -882,9 +882,9 @@ <p><strong>Windows, Triggers, and WindowPanes</strong>: The window operator groups incoming messages in the MessageStream into finite windows. Each emitted result contains one or more messages in the window and is called a WindowPane.</p> -<p>A window can have one or more associated triggers which determine when results from the window are emitted. Triggers can be either <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/windows/Window.html#setEarlyTrigger-org.apache.samza.operators.triggers.Trigger-">early triggers</a> that allow emitting results speculatively before all data for the window has arrived, or late triggers that allow handling late messages for the window.</p> +<p>A window can have one or more associated triggers which determine when results from the window are emitted. Triggers can be either <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/windows/Window.html#setEarlyTrigger-org.apache.samza.operators.triggers.Trigger-">early triggers</a> that allow emitting results speculatively before all data for the window has arrived, or late triggers that allow handling late messages for the window.</p> -<p><strong>Aggregator Function</strong>: By default, the emitted WindowPane will contain all the messages for the window. Instead of retaining all messages, you typically define a more compact data structure for the WindowPane and update it incrementally as new messages arrive, e.g. for keeping a count of messages in the window. To do this, you can provide an aggregating <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/operators/functions/FoldLeftFunction.html">FoldLeftFunction</a> which is invoked for each incoming message added to the window and defines how to update the WindowPane for that message.</p> +<p><strong>Aggregator Function</strong>: By default, the emitted WindowPane will contain all the messages for the window. Instead of retaining all messages, you typically define a more compact data structure for the WindowPane and update it incrementally as new messages arrive, e.g. for keeping a count of messages in the window. To do this, you can provide an aggregating <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/operators/functions/FoldLeftFunction.html">FoldLeftFunction</a> which is invoked for each incoming message added to the window and defines how to update the WindowPane for that message.</p> <p><strong>Accumulation Mode</strong>: A windowâs accumulation mode determines how results emitted from a window relate to previously emitted results for the same window. This is particularly useful when the window is configured with early or late triggers. The accumulation mode can either be discarding or accumulating.</p> @@ -982,7 +982,7 @@ <h4 id="external-partition-management">External Partition Management</h4> -<p>With external partition management, Samza doesnât manage the partitioning by itself. Instead it uses a <code>PassthroughJobCoordinator</code> which honors whatever partition mapping is provided by the <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/container/grouper/stream/SystemStreamPartitionGrouper.html">SystemStreamPartitionGrouper</a>. There are two common patterns for external partition management:</p> +<p>With external partition management, Samza doesnât manage the partitioning by itself. Instead it uses a <code>PassthroughJobCoordinator</code> which honors whatever partition mapping is provided by the <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/container/grouper/stream/SystemStreamPartitionGrouper.html">SystemStreamPartitionGrouper</a>. There are two common patterns for external partition management:</p> <ul> <li><strong>Using high level Kafka consumer</strong> - partition assignment is done by the high level Kafka consumer itself. To use this model, you need to implement and configure a SystemFactory which provides the Kafka high level consumer. Then you need to configure <em>job.systemstreampartition.grouper.factory</em> to <em>org.apache.samza.container.grouper.stream.AllSspToSingleTaskGrouper</em> so Kafka’s partition assignments all go to one task.</li> @@ -1026,7 +1026,7 @@ There is one leader processor which gene <p>The following diagram shows the relationships of the coordinators in the ZooKeeper coordination service implementation.</p> -<p><img src="/img/1.0.0/learn/documentation/introduction/coordination-service.png" alt="Coordination service diagram" style="max-width: 100%; height: auto;" onclick="window.open(this.src)"></p> +<p><img src="/img/latest/learn/documentation/introduction/coordination-service.png" alt="Coordination service diagram" style="max-width: 100%; height: auto;" onclick="window.open(this.src)"></p> <p>Here are a few important details about the coordination service:</p> @@ -1040,7 +1040,7 @@ There is one leader processor which gene <p>Embedded deployment is designed to help users who want more control over the deployment of their application. So it is the user’s responsibility to configure and deploy the processors. In case of ZooKeeper coordination, you also need to configure the URL for an instance of ZooKeeper.</p> -<p>Additionally, each processor requires a unique ID to be used with the coordination service. If location affinity is important, this ID should be unique for each processor on a specific hostname (assuming local Storage services). To address this requirement, Samza uses a <a href="/learn/documentation/1.0.0/api/javadocs/org/apache/samza/runtime/ProcessorIdGenerator.html">ProcessorIdGenerator</a> to provide the ID for each processor. If no generator is explicitly configured, the default one will create a UUID for each processor.</p> +<p>Additionally, each processor requires a unique ID to be used with the coordination service. If location affinity is important, this ID should be unique for each processor on a specific hostname (assuming local Storage services). To address this requirement, Samza uses a <a href="/learn/documentation/latest/api/javadocs/org/apache/samza/runtime/ProcessorIdGenerator.html">ProcessorIdGenerator</a> to provide the ID for each processor. If no generator is explicitly configured, the default one will create a UUID for each processor.</p> <h4 id="configuration">Configuration</h4> @@ -1080,7 +1080,7 @@ There is one leader processor which gene <p>In the code above, <code>WikipediaApplication</code> is an application written with the <a href="#high-level-api">high level API</a>.</p> -<p>Check out the <a href="/learn/tutorials/1.0.0/hello-samza-high-level-zk.html">tutorial</a> to run this application with ZooKeeper coordination on your machine now.</p> +<p>Check out the <a href="/learn/tutorials/latest/hello-samza-high-level-zk.html">tutorial</a> to run this application with ZooKeeper coordination on your machine now.</p> <h4 id="deployment-and-scaling">Deployment and Scaling</h4> @@ -1139,7 +1139,7 @@ There is one leader processor which gene <div class="footer__items"> <a class="footer__item" href="/meetups/">Meetups</a> <a class="footer__item" href="/blog/">Blog</a> - <a class="footer__item" href="/learn/documentation/1.0.0/introduction/background.html">About</a> + <a class="footer__item" href="/learn/documentation/latest/introduction/background.html">About</a> </div> </div> <div> Added: samza/site/startup/quick-start/versioned/beam.html URL: http://svn.apache.org/viewvc/samza/site/startup/quick-start/versioned/beam.html?rev=1855804&view=auto ============================================================================== --- samza/site/startup/quick-start/versioned/beam.html (added) +++ samza/site/startup/quick-start/versioned/beam.html Tue Mar 19 05:31:11 2019 @@ -0,0 +1,701 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + +<head> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> + <meta charset="utf-8"> + <title>Samza - Beam on Samza Quick Start</title> + <link rel="apple-touch-icon-precomposed" sizes="57x57" href="/img/favicon/apple-touch-icon-57x57.png" /> + <link rel="apple-touch-icon-precomposed" sizes="114x114" href="/img/favicon/apple-touch-icon-114x114.png" /> + <link rel="apple-touch-icon-precomposed" sizes="72x72" href="/img/favicon/apple-touch-icon-72x72.png" /> + <link rel="apple-touch-icon-precomposed" sizes="144x144" href="/img/favicon/apple-touch-icon-144x144.png" /> + <link rel="apple-touch-icon-precomposed" sizes="60x60" href="/img/favicon/apple-touch-icon-60x60.png" /> + <link rel="apple-touch-icon-precomposed" sizes="120x120" href="/img/favicon/apple-touch-icon-120x120.png" /> + <link rel="apple-touch-icon-precomposed" sizes="76x76" href="/img/favicon/apple-touch-icon-76x76.png" /> + <link rel="apple-touch-icon-precomposed" sizes="152x152" href="/img/favicon/apple-touch-icon-152x152.png" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-196x196.png" sizes="196x196" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-96x96.png" sizes="96x96" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-32x32.png" sizes="32x32" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-16x16.png" sizes="16x16" /> + <link rel="icon" type="image/png" href="/img/favicon/favicon-128.png" sizes="128x128" /> + <meta name="application-name" content="https://samza.apache.org"; /> + <meta name="msapplication-TileColor" content="#FFFFFF" /> + <meta name="msapplication-TileImage" content="/img/favicon/mstile-144x144.png" /> + <meta name="msapplication-square70x70logo" content="/img/favicon/mstile-70x70.png" /> + <meta name="msapplication-square150x150logo" content="/img/favicon/mstile-150x150.png" /> + <meta name="msapplication-wide310x150logo" content="/img/favicon/mstile-310x150.png" /> + <meta name="msapplication-square310x310logo" content="/img/favicon/mstile-310x310.png" /> + <link href="/css/ionicons.min.css" rel="stylesheet"> + <link href="/css/google-fonts.css" rel="stylesheet"> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="stylesheet" href="/css/main.new.css" /> +</head> + +<body class="page"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<div class="main-navigation" data-plugin="menu"> + <div class="main-navigation__toggle" data-menu-closed> + <i class="icon ion-md-menu"></i> + </div> + <div class="main-navigation__toggle main-navigation__toggle--opened" data-menu-opened> + <i class="icon ion-md-close"></i> + </div> + <div class="main-navigation__inner"> + <div class="main-navigation__logo"> + <a href="/"> + <img class="main-navigation__logo-img" src="/img/samza-logo.png" srcset="/img/samza-logo.png 1x, /img/[email protected] 2x" + alt="Samza Logo" /> + </a> + </div> + <div class="main-navigation__items" data-menu-opened> + <a class="main-navigation__item" href="/">Home</a> + <a class="main-navigation__item" href="/learn/documentation/latest/core-concepts/core-concepts.html">Docs</a> + <a class="main-navigation__item" href="/powered-by/">Powered By</a> + <a class="main-navigation__item" href="/startup/download/">Downloads</a> + <a class="main-navigation__item" href="/blog/">Blog</a> + <div class="main-navigation__item main-navigation__item--group"> + <div class="main-navigation__item-group-title"> + Community + <i class="icon ion-md-arrow-dropdown"></i> + </div> + <div class="main-navigation__item-group-list"> + <a class="main-navigation__item" href="/community/contact-us.html">Contact Us</a> + <a class="main-navigation__item" href="/contribute/contributors-corner.html">Contributor's Corner</a> + <a class="main-navigation__item" href="/community/committers.html">PMC Members and committers</a> + <a class="main-navigation__item" href="/meetups/">Talks and Meetups</a> + </div> + </div> + </div> + </div> +</div> + + <div class="container"> + <div class="container__toggle"> + <i class="icon ion-md-arrow-dropleft-circle container__toggle-icon"></i> + <i class="icon ion-md-arrow-dropright-circle container__toggle-icon container__toggle-icon--opened"></i> + </div> + + <!-- There is only one menu, but made it as a no-output collection to grab data only --> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + + +<div class="side-navigation"> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Getting Started + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/latest/">QuickStart</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/latest/">Code Examples</a> + + + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Documentation + </div> + <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/latest/"> + + + <!-- Handle sub navigation items from data --> + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Releases + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/1.0.0">1.0.0</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.14">0.14</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.13">0.13</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.12">0.12</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.11">0.11</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/releases/0.10">0.10</a> + + + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group"> + + + <!-- Make menu_title, and start items group if needed --> + + <a class="side-navigation__group-title" data-plugin="top-menu" data-match-active="" href="/blog/"> + Blog + </a> + + + <!-- Handle sub navigation items from data --> + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Community + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="" href="/community/contact-us.html">Contact Us</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/contribute/contributors-corner.html">Contributor's Corner</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/contribute/enhancement-proposal.html">Enhancement Proposal</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/community/committers.html">PMC members & Committers</a> + + + <a class="side-navigation__group-item" data-match-active="" href="/meetups/">Talks and Meetups</a> + + + + + <!-- Handle sub nagivation from site collections --> + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + + + + + <!-- Start Group --> + + <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible"> + + + <!-- Make menu_title, and start items group if needed --> + + <div class="side-navigation__group-title"> + <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> + Case Studies + </div> + <div class="side-navigation__group-items " data-sub-menu > + + + <!-- Handle sub navigation items from data --> + + + + <a class="side-navigation__group-item" data-match-active="exact" href="/case-studies/">View All</a> + + + <hr> + + + + + <!-- Handle sub nagivation from site collections --> + + + + + + + + + + + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/ebay" data-match-active="">eBay</a> + + + + + + + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/tripadvisor" data-match-active="">TripAdvisor</a> + + + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/slack" data-match-active="">Slack</a> + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/optimizely" data-match-active="">Optimizely</a> + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/redfin" data-match-active="">Redfin</a> + + + + + + + + + + + + + + <a class="side-navigation__group-item" href="/case-studies/linkedin" data-match-active="">LinkedIn</a> + + + + + + + + + + + <!-- Close sub nav group --> + + </div> + + + <!-- Close menu group --> + </div> + + + +</div> + + + + <div class="section"> + <div class="content"> + + <h2>Beam on Samza Quick Start</h2> + + + + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<p><a href="https://beam.apache.org/";>Apache Beam</a> is an open-source SDK which provides state-of-the-art data processing API and model for both batch and streaming processing pipelines across multiple languages, i.e. Java, Python and Go. By collaborating with Beam, Samza offers the capability of executing Beam API on Samza’s large-scale and stateful streaming engine. Current Samza supports the full Beam <a href="https://beam.apache.org/documentation/runners/capability-matrix/";>Java API</a>, and the support of Python and Go is work-in-progress.</p> + +<h3 id="setting-up-the-word-count-project">Setting up the Word-Count Project</h3> + +<p>To get started, you need to install <a href="">Java 8 SDK</a> as well as <a href="http://maven.apache.org/download.cgi";>Apache Maven</a>. After that, the easiest way to get a copy of the WordCount examples in Beam API is to use the following command to generate a simple Maven project:</p> + +<figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>> mvn archetype:generate <span class="se">\</span> + -DarchetypeGroupId<span class="o">=</span>org.apache.beam <span class="se">\</span> + -DarchetypeArtifactId<span class="o">=</span>beam-sdks-java-maven-archetypes-examples <span class="se">\</span> + -DarchetypeVersion<span class="o">=</span><span class="m">2</span>.11.0 <span class="se">\</span> + -DgroupId<span class="o">=</span>org.example <span class="se">\</span> + -DartifactId<span class="o">=</span>word-count-beam <span class="se">\</span> + -Dversion<span class="o">=</span><span class="s2">"0.1"</span> <span class="se">\</span> + -Dpackage<span class="o">=</span>org.apache.beam.examples <span class="se">\</span> + -DinteractiveMode<span class="o">=</span><span class="nb">false</span></code></pre></figure> + +<p>This command creates a maven project <code>word-count-beam</code> which contains a series of example pipelines that count words in text files:</p> + +<figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>> <span class="nb">cd</span> word-count-beam/ + +> ls src/main/java/org/apache/beam/examples/ +DebuggingWordCount.java WindowedWordCount.java common +MinimalWordCount.java WordCount.java</code></pre></figure> + +<p>Let’s use the MinimalWordCount example to demonstrate how to create a simple Beam pipeline:</p> + +<figure class="highlight"><pre><code class="language-java" data-lang="java"><span></span><span class="kd">public</span> <span class="kd">class</span> <span class="nc">MinimalWordCount</span> <span class="o">{</span> + + <span class="kd">public</span> <span class="kd">static</span> <span class="kt">void</span> <span class="nf">main</span><span class="o">(</span><span class="n">String</span><span class="o">[]</span> <span class="n">args</span><span class="o">)</span> <span class="o">{</span> + <span class="c1">// Create the Pipeline object with the options we defined above</span> + <span class="n">PipelineOptions</span> <span class="n">options</span> <span class="o">=</span> <span class="n">PipelineOptionsFactory</span><span class="o">.</span><span class="na">create</span><span class="o">();</span> + <span class="n">Pipeline</span> <span class="n">p</span> <span class="o">=</span> <span class="n">Pipeline</span><span class="o">.</span><span class="na">create</span><span class="o">(</span><span class="n">options</span><span class="o">);</span> + + <span class="c1">// This example reads a public data set consisting of the complete works of Shakespeare.</span> + <span class="n">p</span><span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">TextIO</span><span class="o">.</span><span class="na">read</span><span class="o">().</span><span class="na">from</span><span class="o">(</span><span class="s">"gs://apache-beam-samples/shakespeare/*"</span><span class="o">))</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span> + <span class="n">FlatMapElements</span><span class="o">.</span><span class="na">into</span><span class="o">(</span><span class="n">TypeDescriptors</span><span class="o">.</span><span class="na">strings</span><span class="o">())</span> + <span class="o">.</span><span class="na">via</span><span class="o">((</span><span class="n">String</span> <span class="n">word</span><span class="o">)</span> <span class="o">-></span> <span class="n">Arrays</span><span class="o">.</span><span class="na">asList</span><span class="o">(</span><span class="n">word</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">"[^\\p{L}]+"</span><span class="o">))))</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Filter</span><span class="o">.</span><span class="na">by</span><span class="o">((</span><span class="n">String</span> <span class="n">word</span><span class="o">)</span> <span class="o">-></span> <span class="o">!</span><span class="n">word</span><span class="o">.</span><span class="na">isEmpty</span><span class="o">()))</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">Count</span><span class="o">.</span><span class="na">perElement</span><span class="o">())</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span> + <span class="n">MapElements</span><span class="o">.</span><span class="na">into</span><span class="o">(</span><span class="n">TypeDescriptors</span><span class="o">.</span><span class="na">strings</span><span class="o">())</span> + <span class="o">.</span><span class="na">via</span><span class="o">(</span> + <span class="o">(</span><span class="n">KV</span><span class="o"><</span><span class="n">String</span><span class="o">,</span> <span class="n">Long</span><span class="o">></span> <span class="n">wordCount</span><span class="o">)</span> <span class="o">-></span> + <span class="n">wordCount</span><span class="o">.</span><span class="na">getKey</span><span class="o">()</span> <span class="o">+</span> <span class="s">": "</span> <span class="o">+</span> <span class="n">wordCount</span><span class="o">.</span><span class="na">getValue</span><span class="o">()))</span> + <span class="o">.</span><span class="na">apply</span><span class="o">(</span><span class="n">TextIO</span><span class="o">.</span><span class="na">write</span><span class="o">().</span><span class="na">to</span><span class="o">(</span><span class="s">"wordcounts"</span><span class="o">));</span> + + <span class="n">p</span><span class="o">.</span><span class="na">run</span><span class="o">().</span><span class="na">waitUntilFinish</span><span class="o">();</span> + <span class="o">}</span> +<span class="o">}</span></code></pre></figure> + +<p>In this example, we first create a Beam <code>Pipeline</code> object to build the graph of transformations to be executed. Then we first use the Read transform to consume a public data set, and split into words. Then we use Beam build-in <code>Count</code> transform and returns the key/value pairs where each key represents a unique element from the input collection, and each value represents the number of times that key appeared in the input collection. Finally we format the results and write them to a file. A detailed walkthrough of the example code can be found <a href="https://beam.apache.org/get-started/wordcount-example/";>here</a>.</p> + +<p>Let’s run the WordCount example with Samza using the following command:</p> + +<figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>>mvn compile exec:java -Dexec.mainClass<span class="o">=</span>org.apache.beam.examples.WordCount <span class="se">\</span> + -Dexec.args<span class="o">=</span><span class="s2">"--inputFile=pom.xml --output=/tmp/counts --runner=SamzaRunner"</span> -Psamza-runner</code></pre></figure> + +<p>After the pipeline finishes, you can check out the output counts files in /tmp folder. Note Beam generates multiple output files for parallel processing. If you prefer a single output, please update the code to use TextIO.write().withoutSharding().</p> + +<figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span></span>>more /tmp/counts* +This: <span class="m">1</span> +When: <span class="m">1</span> +YARN: <span class="m">1</span> +apex: <span class="m">2</span> +apis: <span class="m">2</span> +beam: <span class="m">43</span> +beta: <span class="m">1</span> +code: <span class="m">2</span> +copy: <span class="m">1</span> +...</code></pre></figure> + +<p>For more examples and how to deploy your job in local, standalone and Yarn cluster, you can look at the <a href="/startup/code-examples/latest/beam.html">code examples</a>. Please don’t hesitate to <a href="https://samza.apache.org/community/contact-us.html";>reach out</a> if you encounter any issues.</p> + + + </div> + </div> + + </div> + + + <!-- footer starts here --> + + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<footer> + <div class="footer-inner"> + <div class="side-by-side"> + <div> + <div class="footer__heading">Learn More</div> + <div class="footer__items"> + <a class="footer__item" href="/meetups/">Meetups</a> + <a class="footer__item" href="/blog/">Blog</a> + <a class="footer__item" href="/learn/documentation/latest/introduction/background.html">About</a> + </div> + </div> + <div> + <div class="footer__heading">Community</div> + <div class="footer__items"> + <a class="footer__item" href="/community/contact-us.html">Contact Us</a> + <a class="footer__item" href="/contribute/contributors-corner.html">Contributors' Corner</a> + <a class="footer__item" href="/community/committers.html">PMC members and committers</a> + <a class="footer__item" href="/powered-by/">Powered By</a> + </div> + </div> + + <div> + <div class="quick-links"> + <a class="quick-link" href="/startup/download" target="_blank"> + <i class="icon ion-md-download"></i> + </a> + <a class="quick-link" href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"> + <i class="icon ion-md-code"></i> + </a> + <a class="quick-link" href="https://twitter.com/samzastream"; target="_blank"> + <i class="icon ion-logo-twitter"></i> + </a> + </div> + + <p> + <script>document.write(new Date().getFullYear());</script> © samza.apache.org</p> + </div> + + </div> + </div> + +</footer> + + +<script> + var tryFile = function (url, cb) { + var myRequest = new Request(url); + fetch(myRequest).then((response, cb) => { + console.log(response.status); // returns 200 + cb(response.status != 404); + }); + } + + tryFile(window.location.pathname, function (status) { + // do something with the status + console.log(status); + }); +</script> + + +<!-- Google Analytics --> +<script> + (function (i, s, o, g, r, a, m) { + i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function () { + (i[r].q = i[r].q || []).push(arguments) + }, i[r].l = 1 * new Date(); a = s.createElement(o), + m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g; m.parentNode.insertBefore(a, m) + })(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + +</script> +<script src="/js/main.new.js"></script> + +</body> + +</html> \ No newline at end of file
