Added: samza/site/learn/documentation/0.10/container/checkpointing.html URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.10/container/checkpointing.html?rev=1721445&view=auto ============================================================================== --- samza/site/learn/documentation/0.10/container/checkpointing.html (added) +++ samza/site/learn/documentation/0.10/container/checkpointing.html Tue Dec 22 19:01:05 2015 @@ -0,0 +1,288 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Samza - Checkpointing</title> + <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/> + <link href="/css/bootstrap.min.css" rel="stylesheet"/> + <link href="/css/font-awesome.min.css" rel="stylesheet"/> + <link href="/css/main.css" rel="stylesheet"/> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="icon" type="image/png" href="/img/samza-icon.png"> + <script src="/js/jquery-1.11.1.min.js"></script> + </head> + <body> + <div class="wrapper"> + <div class="wrapper-content"> + + <div class="masthead"> + <div class="container"> + <div class="masthead-logo"> + <a href="/" class="logo">samza</a> + </div> + <div class="masthead-icons"> + <div class="pull-right"> + <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a> + <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a> + <a href="https://twitter.com/samzastream"; target="_blank"><i class="fa fa-twitter masthead-icon"></i></a> + <!-- this icon only shows in versioned pages --> + + + + + <a href="http://samza.apache.org/learn/documentation/latest/container/checkpointing.html";><i id="switch-version-button"></i></a> + <!-- links for the navigation bar --> + + + </div> + </div> + </div><!-- /.container --> + </div> + + <div class="container"> + <div class="menu"> + <h1><i class="fa fa-rocket"></i> Getting Started</h1> + <ul> + <li><a href="/startup/hello-samza/0.10">Hello Samza</a></li> + <li><a href="/startup/download">Download</a></li> + </ul> + + <h1><i class="fa fa-book"></i> Learn</h1> + <ul> + <li><a href="/learn/documentation/0.10">Documentation</a></li> + <li><a href="/learn/documentation/0.10/jobs/configuration-table.html">Configuration</a></li> + <li><a href="/learn/documentation/0.10/api/javadocs/">Javadocs</a></li> + <li><a href="/learn/tutorials/0.10">Tutorials</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ";>FAQ</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza";>Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876";>Papers & Talks</a></li> + <li><a href="http://blogs.apache.org/samza";>Blog</a></li> + </ul> + + <h1><i class="fa fa-comments"></i> Community</h1> + <ul> + <li><a href="/community/mailing-lists.html">Mailing Lists</a></li> + <li><a href="/community/irc.html">IRC</a></li> + <li><a href="https://issues.apache.org/jira/browse/SAMZA";>Bugs</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By";>Powered by</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem";>Ecosystem</a></li> + <li><a href="/community/committers.html">Committers</a></li> + </ul> + + <h1><i class="fa fa-code"></i> Contribute</h1> + <ul> + <li><a href="/contribute/rules.html">Rules</a></li> + <li><a href="/contribute/coding-guide.html">Coding Guide</a></li> + <li><a href="/contribute/projects.html">Projects</a></li> + <li><a href="/contribute/design-documents.html">Design Documents</a></li> + <li><a href="/contribute/code.html">Code</a></li> + <li><a href="https://reviews.apache.org/groups/samza";>Review Board</a></li> + <li><a href="/contribute/tests.html">Tests</a></li> + </ul> + + <h1><i class="fa fa-history"></i> Archive</h1> + <ul> + <li><a href="/archive/index.html#latest">latest</a></li> + <li><a href="/archive/index.html#09">0.9</a></li> + <li><a href="/archive/index.html#08">0.8</a></li> + <li><a href="/archive/index.html#07">0.7</a></li> + </ul> + </div> + + <div class="content"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h2>Checkpointing</h2> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<p>Samza provides fault-tolerant processing of streams: Samza guarantees that messages won’t be lost, even if your job crashes, if a machine dies, if there is a network fault, or something else goes wrong. In order to provide this guarantee, Samza expects the <a href="streams.html">input system</a> to meet the following requirements:</p> + +<ul> +<li>The stream may be sharded into one or more <em>partitions</em>. Each partition is independent from the others, and is replicated across multiple machines (the stream continues to be available, even if a machine fails).</li> +<li>Each partition consists of a sequence of messages in a fixed order. Each message has an <em>offset</em>, which indicates its position in that sequence. Messages are always consumed sequentially within each partition.</li> +<li>A Samza job can start consuming the sequence of messages from any starting offset.</li> +</ul> + +<p>Kafka meets these requirements, but they can also be implemented with other message broker systems.</p> + +<p>As described in the <a href="samza-container.html">section on SamzaContainer</a>, each task instance of your job consumes one partition of an input stream. Each task has a <em>current offset</em> for each input stream: the offset of the next message to be read from that stream partition. Every time a message is read from the stream, the current offset moves forwards.</p> + +<p>If a Samza container fails, it needs to be restarted (potentially on another machine) and resume processing where the failed container left off. In order to enable this, a container periodically checkpoints the current offset for each task instance.</p> + +<p><img src="/img/0.10/learn/documentation/container/checkpointing.svg" alt="Illustration of checkpointing" class="diagram-large"></p> + +<p>When a Samza container starts up, it looks for the most recent checkpoint and starts consuming messages from the checkpointed offsets. If the previous container failed unexpectedly, the most recent checkpoint may be slightly behind the current offsets (i.e. the job may have consumed some more messages since the last checkpoint was written), but we can’t know for sure. In that case, the job may process a few messages again.</p> + +<p>This guarantee is called <em>at-least-once processing</em>: Samza ensures that your job doesn’t miss any messages, even if containers need to be restarted. However, it is possible for your job to see the same message more than once when a container is restarted. We are planning to address this in a future version of Samza, but for now it is just something to be aware of: for example, if you are counting page views, a forcefully killed container could cause events to be slightly over-counted. You can reduce duplication by checkpointing more frequently, at a slight performance cost.</p> + +<p>For checkpoints to be effective, they need to be written somewhere where they will survive faults. Samza allows you to write checkpoints to the file system (using FileSystemCheckpointManager), but that doesn’t help if the machine fails and the container needs to be restarted on another machine. The most common configuration is to use Kafka for checkpointing. You can enable this with the following job configuration:</p> + +<div class="highlight"><pre><code class="jproperties"><span class="c"># The name of your job determines the name under which checkpoints will be stored</span> +<span class="na">job.name</span><span class="o">=</span><span class="s">example-job</span> + +<span class="c"># Define a system called "kafka" for consuming and producing to a Kafka cluster</span> +<span class="na">systems.kafka.samza.factory</span><span class="o">=</span><span class="s">org.apache.samza.system.kafka.KafkaSystemFactory</span> + +<span class="c"># Declare that we want our job's checkpoints to be written to Kafka</span> +<span class="na">task.checkpoint.factory</span><span class="o">=</span><span class="s">org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory</span> +<span class="na">task.checkpoint.system</span><span class="o">=</span><span class="s">kafka</span> + +<span class="c"># By default, a checkpoint is written every 60 seconds. You can change this if you like.</span> +<span class="na">task.commit.ms</span><span class="o">=</span><span class="s">60000</span></code></pre></div> + +<p>In this configuration, Samza writes checkpoints to a separate Kafka topic called __samza_checkpoint_<job-name>_<job-id> (in the example configuration above, the topic would be called __samza_checkpoint_example-job_1). Once per minute, Samza automatically sends a message to this topic, in which the current offsets of the input streams are encoded. When a Samza container starts up, it looks for the most recent offset message in this topic, and loads that checkpoint.</p> + +<p>Sometimes it can be useful to use checkpoints only for some input streams, but not for others. In this case, you can tell Samza to ignore any checkpointed offsets for a particular stream name:</p> + +<div class="highlight"><pre><code class="jproperties"><span class="c"># Ignore any checkpoints for the topic "my-special-topic"</span> +<span class="na">systems.kafka.streams.my-special-topic.samza.reset.offset</span><span class="o">=</span><span class="s">true</span> + +<span class="c"># Always start consuming "my-special-topic" at the oldest available offset</span> +<span class="na">systems.kafka.streams.my-special-topic.samza.offset.default</span><span class="o">=</span><span class="s">oldest</span></code></pre></div> + +<p>The following table explains the meaning of these configuration parameters:</p> + +<table class="table table-condensed table-bordered table-striped"> + <thead> + <tr> + <th>Parameter name</th> + <th>Value</th> + <th>Meaning</th> + </tr> + </thead> + <tbody> + <tr> + <td rowspan="2" class="nowrap">systems.<system>.<br>streams.<stream>.<br>samza.reset.offset</td> + <td>false (default)</td> + <td>When container starts up, resume processing from last checkpoint</td> + </tr> + <tr> + <td>true</td> + <td>Ignore checkpoint (pretend that no checkpoint is present)</td> + </tr> + <tr> + <td rowspan="2" class="nowrap">systems.<system>.<br>streams.<stream>.<br>samza.offset.default</td> + <td>upcoming (default)</td> + <td>When container starts and there is no checkpoint (or the checkpoint is ignored), only process messages that are published after the job is started, but no old messages</td> + </tr> + <tr> + <td>oldest</td> + <td>When container starts and there is no checkpoint (or the checkpoint is ignored), jump back to the oldest available message in the system, and consume all messages from that point onwards (most likely this means repeated processing of messages already seen previously)</td> + </tr> + </tbody> +</table> + +<p>Note that the example configuration above causes your tasks to start consuming from the oldest offset <em>every time a container starts up</em>. This is useful in case you have some in-memory state in your tasks that you need to rebuild from source data in an input stream. If you are using streams in this way, you may also find <a href="streams.html">bootstrap streams</a> useful.</p> + +<h3 id="manipulating-checkpoints-manually">Manipulating Checkpoints Manually</h3> + +<p>If you want to make a one-off change to a job’s consumer offsets, for example to force old messages to be <a href="../jobs/reprocessing.html">processed again</a> with a new version of your code, you can use CheckpointTool to inspect and manipulate the job’s checkpoint. The tool is included in Samza’s <a href="/contribute/code.html">source repository</a>.</p> + +<p>To inspect a job’s latest checkpoint, you need to specify your job’s config file, so that the tool knows which job it is dealing with:</p> + +<div class="highlight"><pre><code class="bash">samza-example/target/bin/checkpoint-tool.sh <span class="se">\</span> + --config-path<span class="o">=</span>file:///path/to/job/config.properties</code></pre></div> + +<p>This command prints out the latest checkpoint in a properties file format. You can save the output to a file, and edit it as you wish. For example, to jump back to the oldest possible point in time, you can set all the offsets to 0. Then you can feed that properties file back into checkpoint-tool.sh and save the modified checkpoint:</p> + +<div class="highlight"><pre><code class="bash">samza-example/target/bin/checkpoint-tool.sh <span class="se">\</span> + --config-path<span class="o">=</span>file:///path/to/job/config.properties <span class="se">\</span> + --new-offsets<span class="o">=</span>file:///path/to/new/offsets.properties</code></pre></div> + +<p>Note that Samza only reads checkpoints on container startup. In order for your checkpoint change to take effect, you need to first stop the job, then save the modified offsets, and then start the job again. If you write a checkpoint while the job is running, it will most likely have no effect.</p> + +<h2 id="state-management-»"><a href="state-management.html">State Management »</a></h2> + + + </div> + </div> + + </div><!-- /.wrapper-content --> + </div><!-- /.wrapper --> + + <div class="footer"> + <div class="container"> + <!-- nothing for now. --> + </div> + </div> + + + <script> + $( document ).ready(function() { + if ( $.fn.urlExists( "/learn/documentation/latest/container/checkpointing.html" ) ) { + $("#switch-version-button").addClass("fa fa-history masthead-icon"); + } + }); + + /* a function to test whether the url exists or not */ + (function( $ ) { + $.fn.urlExists = function(url) { + var http = new XMLHttpRequest(); + http.open('HEAD', url, false); + http.send(); + return http.status != 404; + }; + }( jQuery )); + </script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + + </script> + </body> +</html>
Added: samza/site/learn/documentation/0.10/container/coordinator-stream.html URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.10/container/coordinator-stream.html?rev=1721445&view=auto ============================================================================== --- samza/site/learn/documentation/0.10/container/coordinator-stream.html (added) +++ samza/site/learn/documentation/0.10/container/coordinator-stream.html Tue Dec 22 19:01:05 2015 @@ -0,0 +1,320 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Samza - Coordinator Stream</title> + <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/> + <link href="/css/bootstrap.min.css" rel="stylesheet"/> + <link href="/css/font-awesome.min.css" rel="stylesheet"/> + <link href="/css/main.css" rel="stylesheet"/> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="icon" type="image/png" href="/img/samza-icon.png"> + <script src="/js/jquery-1.11.1.min.js"></script> + </head> + <body> + <div class="wrapper"> + <div class="wrapper-content"> + + <div class="masthead"> + <div class="container"> + <div class="masthead-logo"> + <a href="/" class="logo">samza</a> + </div> + <div class="masthead-icons"> + <div class="pull-right"> + <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a> + <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a> + <a href="https://twitter.com/samzastream"; target="_blank"><i class="fa fa-twitter masthead-icon"></i></a> + <!-- this icon only shows in versioned pages --> + + + + + <a href="http://samza.apache.org/learn/documentation/latest/container/coordinator-stream.html";><i id="switch-version-button"></i></a> + <!-- links for the navigation bar --> + + + </div> + </div> + </div><!-- /.container --> + </div> + + <div class="container"> + <div class="menu"> + <h1><i class="fa fa-rocket"></i> Getting Started</h1> + <ul> + <li><a href="/startup/hello-samza/0.10">Hello Samza</a></li> + <li><a href="/startup/download">Download</a></li> + </ul> + + <h1><i class="fa fa-book"></i> Learn</h1> + <ul> + <li><a href="/learn/documentation/0.10">Documentation</a></li> + <li><a href="/learn/documentation/0.10/jobs/configuration-table.html">Configuration</a></li> + <li><a href="/learn/documentation/0.10/api/javadocs/">Javadocs</a></li> + <li><a href="/learn/tutorials/0.10">Tutorials</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ";>FAQ</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza";>Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876";>Papers & Talks</a></li> + <li><a href="http://blogs.apache.org/samza";>Blog</a></li> + </ul> + + <h1><i class="fa fa-comments"></i> Community</h1> + <ul> + <li><a href="/community/mailing-lists.html">Mailing Lists</a></li> + <li><a href="/community/irc.html">IRC</a></li> + <li><a href="https://issues.apache.org/jira/browse/SAMZA";>Bugs</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By";>Powered by</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem";>Ecosystem</a></li> + <li><a href="/community/committers.html">Committers</a></li> + </ul> + + <h1><i class="fa fa-code"></i> Contribute</h1> + <ul> + <li><a href="/contribute/rules.html">Rules</a></li> + <li><a href="/contribute/coding-guide.html">Coding Guide</a></li> + <li><a href="/contribute/projects.html">Projects</a></li> + <li><a href="/contribute/design-documents.html">Design Documents</a></li> + <li><a href="/contribute/code.html">Code</a></li> + <li><a href="https://reviews.apache.org/groups/samza";>Review Board</a></li> + <li><a href="/contribute/tests.html">Tests</a></li> + </ul> + + <h1><i class="fa fa-history"></i> Archive</h1> + <ul> + <li><a href="/archive/index.html#latest">latest</a></li> + <li><a href="/archive/index.html#09">0.9</a></li> + <li><a href="/archive/index.html#08">0.8</a></li> + <li><a href="/archive/index.html#07">0.7</a></li> + </ul> + </div> + + <div class="content"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h2>Coordinator Stream</h2> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<p>Samza job is completely driven by the job configuration. Thus, job configurations tend to be pretty large. In order to easily serialize such large configs and persist them between job executions, Samza writes all configurations to a durable stream called the <em>Coordinator Stream</em> when a job is submitted.</p> + +<p>A Coordinator Stream is single partitioned stream to which the configurations are written to. It shares the same characteristics as any input stream that can be configured in Samza - ordered, replayable and fault-tolerant. The stream will contain three major types of messages:</p> + +<ol> +<li>Job configuration messages</li> +<li>Task changelog partition assignment messages</li> +<li>Container locality message</li> +</ol> + +<h3 id="coordinator-stream-naming">Coordinator Stream Naming</h3> + +<p>The naming convention is very similar to that of the checkpoint topic that get’s created.</p> +<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="s">"__samza_coordinator_%s_%s"</span> <span class="n">format</span> <span class="o">(</span><span class="n">jobName</span><span class="o">.</span><span class="na">replaceAll</span><span class="o">(</span><span class="s">"_"</span><span class="o">,</span> <span class="s">"-"</span><span class="o">),</span> <span class="n">jobId</span><span class="o">.</span><span class="na">replaceAll</span><span class="o">(</span><span class="s">"_"</span><span class="o">,</span> <span class="s">"-"</span><span class="o">))</span> +</code></pre></div> +<h3 id="coordinator-stream-message-model">Coordinator Stream Message Model</h3> + +<p>Coordinator stream messages are modeled as key/value pairs. The key is a list of well defined fields: <em>version</em>, <em>type</em>, and <em>key</em>. The value is a <em>map</em>. There are some pre-defined fields (such as timestamp, host, etc) for the value map, which are common to all messages.</p> + +<p>The full structure for a CoordinatorStreamMessage is:</p> +<div class="highlight"><pre><code class="language-json" data-lang="json"><span class="err">key</span> <span class="err">=></span> <span class="p">[</span><span class="s2">"<version-number>"</span><span class="p">,</span> <span class="s2">"<message-type>"</span><span class="p">,</span> <span class="s2">"<key>"</span><span class="p">]</span> + +<span class="err">message</span> <span class="err">=></span> <span class="p">{</span> + <span class="nt">"host"</span><span class="p">:</span> <span class="s2">"<hostname>"</span><span class="p">,</span> + <span class="nt">"username"</span><span class="p">:</span> <span class="s2">"<username>"</span><span class="p">,</span> + <span class="nt">"source"</span><span class="p">:</span> <span class="s2">"<source-for-this-message>"</span><span class="p">,</span> + <span class="nt">"timestamp"</span><span class="p">:</span> <span class="err"><timestamp-of-the-message></span><span class="p">,</span> + <span class="nt">"values"</span><span class="p">:</span> <span class="p">{</span> <span class="p">}</span> +<span class="p">}</span> +</code></pre></div> +<p>The messages are essentially serialized and transmitted over the wire as JSON blobs. Hence, for serialization to work correctly, it is very important to not have any unnecessary white spaces. The white spaces in the above JSON blob have been shown for legibility only. </p> + +<p>The most important fields are type, key, and values:</p> + +<ul> +<li>type - defines the kind of message</li> +<li>key - defines a key to associate with the values</li> +<li>values map - defined on a per-message-type basis, and defines a set of values associated with the type</li> +</ul> + +<p>The coordinator stream messages that are currently supported are listed below: +<style> + table th, table td { + text-align: left; + vertical-align: top; + padding: 12px; + border-bottom: 1px solid #ccc; + border-top: 1px solid #ccc; + border-left: 0; + border-right: 0; + }</p> +<div class="highlight"><pre><code class="language-text" data-lang="text"> table td.property, table td.default { + white-space: nowrap; + } + + table th { + background-color: #eee; + } +</code></pre></div> +<p></style> +<table> + <tr> + <th>Message</th> + <th>Type</th> + <th>Key</th> + <th>Values Map</th> + </tr> + <tr> + <td> Configuration Message <br /> + (Applies to all configuration <br /> + options listed in <a href="../jobs/configuration-table.html">Configuration</a>) </td> + <td> set-config </td> + <td> <config-name> </td> + <td> ‘value’ => <config-value> </td> + </tr> + <tr> + <td> Task-ChangelogPartition Assignment Message </td> + <td> set-changelog </td> + <td> <<a href="../api/org/apache/samza/container/TaskName.java">TaskName</a>> </td> + <td> ‘partition’ => <Changelog-Partition-Id> + </td> + </tr> + <tr> + <td> Container Locality Message </td> + <td> set-container-host-assignment </td> + <td> <Container-Id> </td> + <td> ‘hostname’ => <HostName> + </td> + </tr> +</table></p> + +<h3 id="coordinator-stream-writer">Coordinator Stream Writer</h3> + +<p>Samza provides a command line tool to write Job Configuration messages to the coordinator stream. The tool can be used as follows:</p> + +<div class="highlight"><pre><code class="bash">samza-example/target/bin/run-coordinator-stream-writer.sh <span class="se">\</span> + --config-path<span class="o">=</span>file:///path/to/job/config.properties <span class="se">\</span> + --type <span class="nb">set</span>-config <span class="se">\</span> + --key yarn.container.count <span class="se">\</span> + --value 8</code></pre></div> + +<h2 id="job-coordinator"><a name="JobCoordinator"></a>Job Coordinator</h2> + +<p>The Job Coordinator bootstraps configuration from the coordinator stream each time upon job start-up. It periodically catches up with any new data written to the coordinator stream and updates the <em>Job Model</em>.</p> + +<p>Job Model is the data model used to represent a Samza job, which also incorporates the Job configuration. The hierarchy of a Samza job - job has containers, and each of the containers has tasks - is encapsulated in the Job Model, along with relevant information such as container id, task names, partition information, etc.</p> + +<p>The Job Coordinator exposes the Job Model and Job Configuration via an HTTP service. The URL for the Job Coordinator’s HTTP service is passed as an environment variable to the Samza Containers when the containers are launched. Containers may write meta-information, such as locality - the hostname of the machine on which the container is running. However, they will read the Job Model and Configuration by querying the Job Coordinator via the HTTP service.</p> + +<p>Thus, Job Coorindator is the single component that has the latest view of the entire job status. This is very useful as it allows us to extend functionality of the Job Coordinator, in the future, to manage the lifecycle of the job (such as start/stop container, modify task assignment etc).</p> + +<h3 id="job-coordinator-availability">Job Coordinator Availability</h3> + +<p>The Job Coordinator resides in the same container as the Samza Application Master. Thus, the availability of the Job Coordinator is tied to the availability of the Application Master (AM) in the Yarn cluster. The Samza containers are started only after initializing the Job Coordinator from the Coordinator Stream. In stable condition, when the Samza container comes up, it should be able to read the JobModel from the Job Coordinator without timing out. </p> + +<h2 id="benefits-of-coordinator-stream-model">Benefits of Coordinator Stream Model</h2> + +<p>Writing the configuration to a durable stream opens the door for Samza to do a couple of things:</p> + +<ol> +<li>Removes the size-bound on the Job configuration</li> +<li>Exposes job-related configuration and metadata to the containers using a standard data model and communication interface (See <a href="#JobCoordinator">Job Coordinator</a> for details)</li> +<li>Certain configurations should only be set one time. Changing them in future deployment amounts to resetting the entire state of the job because it may re-shuffle input partitions to the containers. For example, changing <a href="../api/javadocs/org/apache/samza/container/grouper/stream/SystemStreamPartitionGrouper.java">SystemStreamPartitionGrouper</a> on a stateful Samza job would inter-mingle state from different StreamTasks in a single changelog partition. Without persistent configuration, there is no easy way to check whether a job’s current configuration is valid or not.</li> +<li>Job configuration can be dynamically changed by writing to the Coorinator Stream. This can enable features that require the job to be reactive to configuration change (eg. host-affinity, auto-scaling, dynamic reconfiguration etc).</li> +<li>Provides a unified view of the job state, enabling Samza with more powerful ways of controlling container controls (See <a href="#JobCoordinator">Job Coordinator</a> for details)</li> +<li>Enables future design of Job Coordinator fail-over since it serves as a single source of truth of the current job state</li> +</ol> + +<p>For other interesting features that can leverage this model, please refer the <a href="https://issues.apache.org/jira/secure/attachment/12670650/DESIGN-SAMZA-348-1.pdf";>design document</a>.</p> + + + </div> + </div> + + </div><!-- /.wrapper-content --> + </div><!-- /.wrapper --> + + <div class="footer"> + <div class="container"> + <!-- nothing for now. --> + </div> + </div> + + + <script> + $( document ).ready(function() { + if ( $.fn.urlExists( "/learn/documentation/latest/container/coordinator-stream.html" ) ) { + $("#switch-version-button").addClass("fa fa-history masthead-icon"); + } + }); + + /* a function to test whether the url exists or not */ + (function( $ ) { + $.fn.urlExists = function(url) { + var http = new XMLHttpRequest(); + http.open('HEAD', url, false); + http.send(); + return http.status != 404; + }; + }( jQuery )); + </script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + + </script> + </body> +</html> Added: samza/site/learn/documentation/0.10/container/event-loop.html URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.10/container/event-loop.html?rev=1721445&view=auto ============================================================================== --- samza/site/learn/documentation/0.10/container/event-loop.html (added) +++ samza/site/learn/documentation/0.10/container/event-loop.html Tue Dec 22 19:01:05 2015 @@ -0,0 +1,220 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Samza - Event Loop</title> + <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/> + <link href="/css/bootstrap.min.css" rel="stylesheet"/> + <link href="/css/font-awesome.min.css" rel="stylesheet"/> + <link href="/css/main.css" rel="stylesheet"/> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="icon" type="image/png" href="/img/samza-icon.png"> + <script src="/js/jquery-1.11.1.min.js"></script> + </head> + <body> + <div class="wrapper"> + <div class="wrapper-content"> + + <div class="masthead"> + <div class="container"> + <div class="masthead-logo"> + <a href="/" class="logo">samza</a> + </div> + <div class="masthead-icons"> + <div class="pull-right"> + <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a> + <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a> + <a href="https://twitter.com/samzastream"; target="_blank"><i class="fa fa-twitter masthead-icon"></i></a> + <!-- this icon only shows in versioned pages --> + + + + + <a href="http://samza.apache.org/learn/documentation/latest/container/event-loop.html";><i id="switch-version-button"></i></a> + <!-- links for the navigation bar --> + + + </div> + </div> + </div><!-- /.container --> + </div> + + <div class="container"> + <div class="menu"> + <h1><i class="fa fa-rocket"></i> Getting Started</h1> + <ul> + <li><a href="/startup/hello-samza/0.10">Hello Samza</a></li> + <li><a href="/startup/download">Download</a></li> + </ul> + + <h1><i class="fa fa-book"></i> Learn</h1> + <ul> + <li><a href="/learn/documentation/0.10">Documentation</a></li> + <li><a href="/learn/documentation/0.10/jobs/configuration-table.html">Configuration</a></li> + <li><a href="/learn/documentation/0.10/api/javadocs/">Javadocs</a></li> + <li><a href="/learn/tutorials/0.10">Tutorials</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ";>FAQ</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza";>Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876";>Papers & Talks</a></li> + <li><a href="http://blogs.apache.org/samza";>Blog</a></li> + </ul> + + <h1><i class="fa fa-comments"></i> Community</h1> + <ul> + <li><a href="/community/mailing-lists.html">Mailing Lists</a></li> + <li><a href="/community/irc.html">IRC</a></li> + <li><a href="https://issues.apache.org/jira/browse/SAMZA";>Bugs</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By";>Powered by</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem";>Ecosystem</a></li> + <li><a href="/community/committers.html">Committers</a></li> + </ul> + + <h1><i class="fa fa-code"></i> Contribute</h1> + <ul> + <li><a href="/contribute/rules.html">Rules</a></li> + <li><a href="/contribute/coding-guide.html">Coding Guide</a></li> + <li><a href="/contribute/projects.html">Projects</a></li> + <li><a href="/contribute/design-documents.html">Design Documents</a></li> + <li><a href="/contribute/code.html">Code</a></li> + <li><a href="https://reviews.apache.org/groups/samza";>Review Board</a></li> + <li><a href="/contribute/tests.html">Tests</a></li> + </ul> + + <h1><i class="fa fa-history"></i> Archive</h1> + <ul> + <li><a href="/archive/index.html#latest">latest</a></li> + <li><a href="/archive/index.html#09">0.9</a></li> + <li><a href="/archive/index.html#08">0.8</a></li> + <li><a href="/archive/index.html#07">0.7</a></li> + </ul> + </div> + + <div class="content"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h2>Event Loop</h2> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<p>The event loop is the <a href="samza-container.html">container</a>’s single thread that is in charge of <a href="streams.html">reading and writing messages</a>, <a href="metrics.html">flushing metrics</a>, <a href="checkpointing.html">checkpointing</a>, and <a href="windowing.html">windowing</a>.</p> + +<p>Samza uses a single thread because every container is designed to use a single CPU core; to get more parallelism, simply run more containers. This uses a bit more memory than multithreaded parallelism, because each JVM has some overhead, but it simplifies resource management and improves isolation between jobs. This helps Samza jobs run reliably on a multitenant cluster, where many different jobs written by different people are running at the same time.</p> + +<p>You are strongly discouraged from using threads in your job’s code. Samza uses multiple threads internally for communicating with input and output streams, but all message processing and user code runs on a single-threaded event loop. In general, Samza is not thread-safe.</p> + +<h3 id="event-loop-internals">Event Loop Internals</h3> + +<p>A container may have multiple <a href="../api/javadocs/org/apache/samza/system/SystemConsumer.html">SystemConsumers</a> for consuming messages from different input systems. Each SystemConsumer reads messages on its own thread, but writes messages into a shared in-process message queue. The container uses this queue to funnel all of the messages into the event loop.</p> + +<p>The event loop works as follows:</p> + +<ol> +<li>Take a message from the incoming message queue;</li> +<li>Give the message to the appropriate <a href="samza-container.html">task instance</a> by calling process() on it;</li> +<li>Call window() on the task instance if it implements <a href="../api/javadocs/org/apache/samza/task/WindowableTask.html">WindowableTask</a>, and the window time has expired;</li> +<li>Send any output from the process() and window() calls to the appropriate <a href="../api/javadocs/org/apache/samza/system/SystemProducer.html">SystemProducers</a>;</li> +<li>Write checkpoints for any tasks whose <a href="checkpointing.html">commit interval</a> has elapsed.</li> +</ol> + +<p>The container does this, in a loop, until it is shut down. Note that although there can be multiple task instances within a container (depending on the number of input stream partitions), their process() and window() methods are all called on the same thread, never concurrently on different threads.</p> + +<h3 id="lifecycle">Lifecycle</h3> + +<p>The only way in which a developer can hook into a SamzaContainer’s lifecycle is through the standard InitableTask, ClosableTask, StreamTask, and WindowableTask. In cases where pluggable logic needs to be added to wrap a StreamTask, the StreamTask can be wrapped by another StreamTask implementation that handles the custom logic before calling into the wrapped StreamTask.</p> + +<p>A concrete example is a set of StreamTasks that all want to share the same try/catch logic in their process() method. A StreamTask can be implemented that wraps the original StreamTasks, and surrounds the original process() call with the appropriate try/catch logic. For more details, see <a href="https://issues.apache.org/jira/browse/SAMZA-437";>this discussion</a>.</p> + +<h2 id="jmx-»"><a href="jmx.html">JMX »</a></h2> + + + </div> + </div> + + </div><!-- /.wrapper-content --> + </div><!-- /.wrapper --> + + <div class="footer"> + <div class="container"> + <!-- nothing for now. --> + </div> + </div> + + + <script> + $( document ).ready(function() { + if ( $.fn.urlExists( "/learn/documentation/latest/container/event-loop.html" ) ) { + $("#switch-version-button").addClass("fa fa-history masthead-icon"); + } + }); + + /* a function to test whether the url exists or not */ + (function( $ ) { + $.fn.urlExists = function(url) { + var http = new XMLHttpRequest(); + http.open('HEAD', url, false); + http.send(); + return http.status != 404; + }; + }( jQuery )); + </script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + + </script> + </body> +</html> Added: samza/site/learn/documentation/0.10/container/jmx.html URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.10/container/jmx.html?rev=1721445&view=auto ============================================================================== --- samza/site/learn/documentation/0.10/container/jmx.html (added) +++ samza/site/learn/documentation/0.10/container/jmx.html Tue Dec 22 19:01:05 2015 @@ -0,0 +1,207 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Samza - JMX</title> + <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/> + <link href="/css/bootstrap.min.css" rel="stylesheet"/> + <link href="/css/font-awesome.min.css" rel="stylesheet"/> + <link href="/css/main.css" rel="stylesheet"/> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="icon" type="image/png" href="/img/samza-icon.png"> + <script src="/js/jquery-1.11.1.min.js"></script> + </head> + <body> + <div class="wrapper"> + <div class="wrapper-content"> + + <div class="masthead"> + <div class="container"> + <div class="masthead-logo"> + <a href="/" class="logo">samza</a> + </div> + <div class="masthead-icons"> + <div class="pull-right"> + <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a> + <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a> + <a href="https://twitter.com/samzastream"; target="_blank"><i class="fa fa-twitter masthead-icon"></i></a> + <!-- this icon only shows in versioned pages --> + + + + + <a href="http://samza.apache.org/learn/documentation/latest/container/jmx.html";><i id="switch-version-button"></i></a> + <!-- links for the navigation bar --> + + + </div> + </div> + </div><!-- /.container --> + </div> + + <div class="container"> + <div class="menu"> + <h1><i class="fa fa-rocket"></i> Getting Started</h1> + <ul> + <li><a href="/startup/hello-samza/0.10">Hello Samza</a></li> + <li><a href="/startup/download">Download</a></li> + </ul> + + <h1><i class="fa fa-book"></i> Learn</h1> + <ul> + <li><a href="/learn/documentation/0.10">Documentation</a></li> + <li><a href="/learn/documentation/0.10/jobs/configuration-table.html">Configuration</a></li> + <li><a href="/learn/documentation/0.10/api/javadocs/">Javadocs</a></li> + <li><a href="/learn/tutorials/0.10">Tutorials</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ";>FAQ</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza";>Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876";>Papers & Talks</a></li> + <li><a href="http://blogs.apache.org/samza";>Blog</a></li> + </ul> + + <h1><i class="fa fa-comments"></i> Community</h1> + <ul> + <li><a href="/community/mailing-lists.html">Mailing Lists</a></li> + <li><a href="/community/irc.html">IRC</a></li> + <li><a href="https://issues.apache.org/jira/browse/SAMZA";>Bugs</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By";>Powered by</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem";>Ecosystem</a></li> + <li><a href="/community/committers.html">Committers</a></li> + </ul> + + <h1><i class="fa fa-code"></i> Contribute</h1> + <ul> + <li><a href="/contribute/rules.html">Rules</a></li> + <li><a href="/contribute/coding-guide.html">Coding Guide</a></li> + <li><a href="/contribute/projects.html">Projects</a></li> + <li><a href="/contribute/design-documents.html">Design Documents</a></li> + <li><a href="/contribute/code.html">Code</a></li> + <li><a href="https://reviews.apache.org/groups/samza";>Review Board</a></li> + <li><a href="/contribute/tests.html">Tests</a></li> + </ul> + + <h1><i class="fa fa-history"></i> Archive</h1> + <ul> + <li><a href="/archive/index.html#latest">latest</a></li> + <li><a href="/archive/index.html#09">0.9</a></li> + <li><a href="/archive/index.html#08">0.8</a></li> + <li><a href="/archive/index.html#07">0.7</a></li> + </ul> + </div> + + <div class="content"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h2>JMX</h2> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<p>Samza’s containers and YARN ApplicationMaster enable <a href="http://docs.oracle.com/javase/tutorial/jmx/";>JMX</a> by default. JMX can be used for managing the JVM; for example, you can connect to it using <a href="http://docs.oracle.com/javase/7/docs/technotes/guides/management/jconsole.html";>jconsole</a>, which is included in the JDK.</p> + +<p>You can tell Samza to publish its internal <a href="metrics.html">metrics</a>, and any custom metrics you define, as JMX MBeans. To enable this, set the following properties in your job configuration:</p> + +<div class="highlight"><pre><code class="jproperties"><span class="c"># Define a Samza metrics reporter called "jmx", which publishes to JMX</span> +<span class="na">metrics.reporter.jmx.class</span><span class="o">=</span><span class="s">org.apache.samza.metrics.reporter.JmxReporterFactory</span> + +<span class="c"># Use it (if you have multiple reporters defined, separate them with commas)</span> +<span class="na">metrics.reporters</span><span class="o">=</span><span class="s">jmx</span></code></pre></div> + +<p>JMX needs to be configured to use a specific port, but in a distributed environment, there is no way of knowing in advance which ports are available on the machines running your containers. Therefore Samza chooses the JMX port randomly. If you need to connect to it, you can find the port by looking in the container’s logs, which report the JMX server details as follows:</p> +<div class="highlight"><pre><code class="language-text" data-lang="text">2014-06-02 21:50:17 JmxServer [INFO] According to InetAddress.getLocalHost.getHostName we are samza-grid-1234.example.com +2014-06-02 21:50:17 JmxServer [INFO] Started JmxServer registry port=50214 server port=50215 url=service:jmx:rmi://localhost:50215/jndi/rmi://localhost:50214/jmxrmi +2014-06-02 21:50:17 JmxServer [INFO] If you are tunneling, you might want to try JmxServer registry port=50214 server port=50215 url=service:jmx:rmi://samza-grid-1234.example.com:50215/jndi/rmi://samza-grid-1234.example.com:50214/jmxrmi +</code></pre></div> +<h2 id="jobrunner-»"><a href="../jobs/job-runner.html">JobRunner »</a></h2> + + + </div> + </div> + + </div><!-- /.wrapper-content --> + </div><!-- /.wrapper --> + + <div class="footer"> + <div class="container"> + <!-- nothing for now. --> + </div> + </div> + + + <script> + $( document ).ready(function() { + if ( $.fn.urlExists( "/learn/documentation/latest/container/jmx.html" ) ) { + $("#switch-version-button").addClass("fa fa-history masthead-icon"); + } + }); + + /* a function to test whether the url exists or not */ + (function( $ ) { + $.fn.urlExists = function(url) { + var http = new XMLHttpRequest(); + http.open('HEAD', url, false); + http.send(); + return http.status != 404; + }; + }( jQuery )); + </script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + + </script> + </body> +</html> Added: samza/site/learn/documentation/0.10/container/metrics.html URL: http://svn.apache.org/viewvc/samza/site/learn/documentation/0.10/container/metrics.html?rev=1721445&view=auto ============================================================================== --- samza/site/learn/documentation/0.10/container/metrics.html (added) +++ samza/site/learn/documentation/0.10/container/metrics.html Tue Dec 22 19:01:05 2015 @@ -0,0 +1,266 @@ +<!DOCTYPE html> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<html lang="en"> + <head> + <meta charset="utf-8"> + <title>Samza - Metrics</title> + <link href='/css/ropa-sans.css' rel='stylesheet' type='text/css'/> + <link href="/css/bootstrap.min.css" rel="stylesheet"/> + <link href="/css/font-awesome.min.css" rel="stylesheet"/> + <link href="/css/main.css" rel="stylesheet"/> + <link href="/css/syntax.css" rel="stylesheet"/> + <link rel="icon" type="image/png" href="/img/samza-icon.png"> + <script src="/js/jquery-1.11.1.min.js"></script> + </head> + <body> + <div class="wrapper"> + <div class="wrapper-content"> + + <div class="masthead"> + <div class="container"> + <div class="masthead-logo"> + <a href="/" class="logo">samza</a> + </div> + <div class="masthead-icons"> + <div class="pull-right"> + <a href="/startup/download"><i class="fa fa-arrow-circle-o-down masthead-icon"></i></a> + <a href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree"; target="_blank"><i class="fa fa-code masthead-icon" style="font-weight: bold;"></i></a> + <a href="https://twitter.com/samzastream"; target="_blank"><i class="fa fa-twitter masthead-icon"></i></a> + <!-- this icon only shows in versioned pages --> + + + + + <a href="http://samza.apache.org/learn/documentation/latest/container/metrics.html";><i id="switch-version-button"></i></a> + <!-- links for the navigation bar --> + + + </div> + </div> + </div><!-- /.container --> + </div> + + <div class="container"> + <div class="menu"> + <h1><i class="fa fa-rocket"></i> Getting Started</h1> + <ul> + <li><a href="/startup/hello-samza/0.10">Hello Samza</a></li> + <li><a href="/startup/download">Download</a></li> + </ul> + + <h1><i class="fa fa-book"></i> Learn</h1> + <ul> + <li><a href="/learn/documentation/0.10">Documentation</a></li> + <li><a href="/learn/documentation/0.10/jobs/configuration-table.html">Configuration</a></li> + <li><a href="/learn/documentation/0.10/api/javadocs/">Javadocs</a></li> + <li><a href="/learn/tutorials/0.10">Tutorials</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/FAQ";>FAQ</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Apache+Samza";>Wiki</a></li> + <li><a href="https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=51812876";>Papers & Talks</a></li> + <li><a href="http://blogs.apache.org/samza";>Blog</a></li> + </ul> + + <h1><i class="fa fa-comments"></i> Community</h1> + <ul> + <li><a href="/community/mailing-lists.html">Mailing Lists</a></li> + <li><a href="/community/irc.html">IRC</a></li> + <li><a href="https://issues.apache.org/jira/browse/SAMZA";>Bugs</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Powered+By";>Powered by</a></li> + <li><a href="https://cwiki.apache.org/confluence/display/SAMZA/Ecosystem";>Ecosystem</a></li> + <li><a href="/community/committers.html">Committers</a></li> + </ul> + + <h1><i class="fa fa-code"></i> Contribute</h1> + <ul> + <li><a href="/contribute/rules.html">Rules</a></li> + <li><a href="/contribute/coding-guide.html">Coding Guide</a></li> + <li><a href="/contribute/projects.html">Projects</a></li> + <li><a href="/contribute/design-documents.html">Design Documents</a></li> + <li><a href="/contribute/code.html">Code</a></li> + <li><a href="https://reviews.apache.org/groups/samza";>Review Board</a></li> + <li><a href="/contribute/tests.html">Tests</a></li> + </ul> + + <h1><i class="fa fa-history"></i> Archive</h1> + <ul> + <li><a href="/archive/index.html#latest">latest</a></li> + <li><a href="/archive/index.html#09">0.9</a></li> + <li><a href="/archive/index.html#08">0.8</a></li> + <li><a href="/archive/index.html#07">0.7</a></li> + </ul> + </div> + + <div class="content"> + <!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<h2>Metrics</h2> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<p>When you’re running a stream process in production, it’s important that you have good metrics to track the health of your job. In order to make this easy, Samza includes a metrics library. It is used by Samza itself to generate some standard metrics such as message throughput, but you can also use it in your task code to emit custom metrics.</p> + +<p>Metrics can be reported in various ways. You can expose them via <a href="jmx.html">JMX</a>, which is useful in development. In production, a common setup is for each Samza container to periodically publish its metrics to a “metrics” Kafka topic, in which the metrics from all Samza jobs are aggregated. You can then consume this stream in another Samza job, and send the metrics to your favorite graphing system such as <a href="http://graphite.wikidot.com/";>Graphite</a>.</p> + +<p>To set up your job to publish metrics to Kafka, you can use the following configuration:</p> + +<div class="highlight"><pre><code class="jproperties"><span class="c"># Define a metrics reporter called "snapshot", which publishes metrics</span> +<span class="c"># every 60 seconds.</span> +<span class="na">metrics.reporters</span><span class="o">=</span><span class="s">snapshot</span> +<span class="na">metrics.reporter.snapshot.class</span><span class="o">=</span><span class="s">org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory</span> + +<span class="c"># Tell the snapshot reporter to publish to a topic called "metrics"</span> +<span class="c"># in the "kafka" system.</span> +<span class="na">metrics.reporter.snapshot.stream</span><span class="o">=</span><span class="s">kafka.metrics</span> + +<span class="c"># Encode metrics data as JSON.</span> +<span class="na">serializers.registry.metrics.class</span><span class="o">=</span><span class="s">org.apache.samza.serializers.MetricsSnapshotSerdeFactory</span> +<span class="na">systems.kafka.streams.metrics.samza.msg.serde</span><span class="o">=</span><span class="s">metrics</span></code></pre></div> + +<p>With this configuration, the job automatically sends several JSON-encoded messages to the “metrics” topic in Kafka every 60 seconds. The messages look something like this:</p> + +<div class="highlight"><pre><code class="json"><span class="p">{</span> + <span class="nt">"header"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"container-name"</span><span class="p">:</span> <span class="s2">"samza-container-0"</span><span class="p">,</span> + <span class="nt">"host"</span><span class="p">:</span> <span class="s2">"samza-grid-1234.example.com"</span><span class="p">,</span> + <span class="nt">"job-id"</span><span class="p">:</span> <span class="s2">"1"</span><span class="p">,</span> + <span class="nt">"job-name"</span><span class="p">:</span> <span class="s2">"my-samza-job"</span><span class="p">,</span> + <span class="nt">"reset-time"</span><span class="p">:</span> <span class="mi">1401729000347</span><span class="p">,</span> + <span class="nt">"samza-version"</span><span class="p">:</span> <span class="s2">"0.0.1"</span><span class="p">,</span> + <span class="nt">"source"</span><span class="p">:</span> <span class="s2">"Partition-2"</span><span class="p">,</span> + <span class="nt">"time"</span><span class="p">:</span> <span class="mi">1401729420566</span><span class="p">,</span> + <span class="nt">"version"</span><span class="p">:</span> <span class="s2">"0.0.1"</span> + <span class="p">},</span> + <span class="nt">"metrics"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"org.apache.samza.container.TaskInstanceMetrics"</span><span class="p">:</span> <span class="p">{</span> + <span class="nt">"commit-calls"</span><span class="p">:</span> <span class="mi">7</span><span class="p">,</span> + <span class="nt">"commit-skipped"</span><span class="p">:</span> <span class="mi">77948</span><span class="p">,</span> + <span class="nt">"kafka-input-topic-offset"</span><span class="p">:</span> <span class="s2">"1606"</span><span class="p">,</span> + <span class="nt">"messages-sent"</span><span class="p">:</span> <span class="mi">985</span><span class="p">,</span> + <span class="nt">"process-calls"</span><span class="p">:</span> <span class="mi">1093</span><span class="p">,</span> + <span class="nt">"send-calls"</span><span class="p">:</span> <span class="mi">985</span><span class="p">,</span> + <span class="nt">"send-skipped"</span><span class="p">:</span> <span class="mi">76970</span><span class="p">,</span> + <span class="nt">"window-calls"</span><span class="p">:</span> <span class="mi">0</span><span class="p">,</span> + <span class="nt">"window-skipped"</span><span class="p">:</span> <span class="mi">77955</span> + <span class="p">}</span> + <span class="p">}</span> +<span class="p">}</span></code></pre></div> + +<p>There is a separate message for each task instance, and the header tells you the job name, job ID and partition of the task. The metrics allow you to see how many messages have been processed and sent, the current offset in the input stream partition, and other details. There are additional messages which give you metrics about the JVM (heap size, garbage collection information, threads etc.), internal metrics of the Kafka producers and consumers, and more.</p> + +<p>It’s easy to generate custom metrics in your job, if there’s some value you want to keep an eye on. You can use Samza’s built-in metrics framework, which is similar in design to Coda Hale’s <a href="http://metrics.codahale.com/";>metrics</a> library. </p> + +<p>You can register your custom metrics through a <a href="../api/javadocs/org/apache/samza/metrics/MetricsRegistry.html">MetricsRegistry</a>. Your stream task needs to implement <a href="../api/javadocs/org/apache/samza/task/InitableTask.html">InitableTask</a>, so that you can get the metrics registry from the <a href="../api/javadocs/org/apache/samza/task/TaskContext.html">TaskContext</a>. This simple example shows how to count the number of messages processed by your task:</p> + +<div class="highlight"><pre><code class="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">MyJavaStreamTask</span> <span class="kd">implements</span> <span class="n">StreamTask</span><span class="o">,</span> <span class="n">InitableTask</span> <span class="o">{</span> + <span class="kd">private</span> <span class="n">Counter</span> <span class="n">messageCount</span><span class="o">;</span> + + <span class="kd">public</span> <span class="kt">void</span> <span class="nf">init</span><span class="o">(</span><span class="n">Config</span> <span class="n">config</span><span class="o">,</span> <span class="n">TaskContext</span> <span class="n">context</span><span class="o">)</span> <span class="o">{</span> + <span class="k">this</span><span class="o">.</span><span class="na">messageCount</span> <span class="o">=</span> <span class="n">context</span> + <span class="o">.</span><span class="na">getMetricsRegistry</span><span class="o">()</span> + <span class="o">.</span><span class="na">newCounter</span><span class="o">(</span><span class="n">getClass</span><span class="o">().</span><span class="na">getName</span><span class="o">(),</span> <span class="s">"message-count"</span><span class="o">);</span> + <span class="o">}</span> + + <span class="kd">public</span> <span class="kt">void</span> <span class="nf">process</span><span class="o">(</span><span class="n">IncomingMessageEnvelope</span> <span class="n">envelope</span><span class="o">,</span> + <span class="n">MessageCollector</span> <span class="n">collector</span><span class="o">,</span> + <span class="n">TaskCoordinator</span> <span class="n">coordinator</span><span class="o">)</span> <span class="o">{</span> + <span class="n">messageCount</span><span class="o">.</span><span class="na">inc</span><span class="o">();</span> + <span class="o">}</span> +<span class="o">}</span></code></pre></div> + +<p>Samza currently supports three kinds of metrics: <a href="../api/javadocs/org/apache/samza/metrics/Counter.html">counters</a>, <a href="../api/javadocs/org/apache/samza/metrics/Gauge.html">gauges</a> and <a href="../api/javadocs/org/apache/samza/metrics/Timer.html">timer</a>. Use a counter when you want to track how often something occurs, a gauge when you want to report the level of something, such as the size of a buffer, and a timer when you want to know how much time the block of code spends. Each task instance (for each input stream partition) gets its own set of metrics.</p> + +<p>If you want to report metrics in some other way, e.g. directly to a graphing system (without going via Kafka), you can implement a <a href="../api/javadocs/org/apache/samza/metrics/MetricsReporterFactory.html">MetricsReporterFactory</a> and reference it in your job configuration.</p> + +<h2 id="windowing-»"><a href="windowing.html">Windowing »</a></h2> + + + </div> + </div> + + </div><!-- /.wrapper-content --> + </div><!-- /.wrapper --> + + <div class="footer"> + <div class="container"> + <!-- nothing for now. --> + </div> + </div> + + + <script> + $( document ).ready(function() { + if ( $.fn.urlExists( "/learn/documentation/latest/container/metrics.html" ) ) { + $("#switch-version-button").addClass("fa fa-history masthead-icon"); + } + }); + + /* a function to test whether the url exists or not */ + (function( $ ) { + $.fn.urlExists = function(url) { + var http = new XMLHttpRequest(); + http.open('HEAD', url, false); + http.send(); + return http.status != 404; + }; + }( jQuery )); + </script> + + + <!-- Google Analytics --> + <script> + (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ + (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), + m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) + })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); + + ga('create', 'UA-43122768-1', 'apache.org'); + ga('send', 'pageview'); + + </script> + </body> +</html>
