http://git-wip-us.apache.org/repos/asf/storm-site/blob/ff14ea94/content/releases/0.10.1/Documentation.html ---------------------------------------------------------------------- diff --git a/content/releases/0.10.1/Documentation.html b/content/releases/0.10.1/Documentation.html deleted file mode 100644 index f09588d..0000000 --- a/content/releases/0.10.1/Documentation.html +++ /dev/null @@ -1,307 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <title>Apache Storm</title> - - <!-- Bootstrap core CSS --> - <link href="/assets/css/bootstrap.min.css" rel="stylesheet"> - <!-- Bootstrap theme --> - <link href="/assets/css/bootstrap-theme.min.css" rel="stylesheet"> - - <!-- Custom styles for this template --> - <link rel="stylesheet" href="http://fortawesome.github.io/Font-Awesome/assets/font-awesome/css/font-awesome.css";> - <link href="/css/style.css" rel="stylesheet"> - <link href="/assets/css/owl.theme.css" rel="stylesheet"> - <link href="/assets/css/owl.carousel.css" rel="stylesheet"> - <script type="text/javascript" src="/assets/js/jquery.min.js"></script> - <script type="text/javascript" src="/assets/js/bootstrap.min.js"></script> - <script type="text/javascript" src="/assets/js/owl.carousel.min.js"></script> - <script type="text/javascript" src="/assets/js/storm.js"></script> - <!-- Just for debugging purposes. Don't actually copy these 2 lines! --> - <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]--> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - - - <body> - <header> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-5"> - <a href="/index.html"><img src="/images/logo.png" class="logo" /></a> - </div> - <div class="col-md-5"> - - <h1>Version: 0.10.1</h1> - - </div> - <div class="col-md-2"> - <a href="/downloads.html" class="btn-std btn-block btn-download">Download</a> - </div> - </div> - </div> -</header> -<!--Header End--> -<!--Navigation Begin--> -<div class="navbar" role="banner"> - <div class="container-fluid"> - <div class="navbar-header"> - <button class="navbar-toggle" type="button" data-toggle="collapse" data-target=".bs-navbar-collapse"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - </div> - <nav class="collapse navbar-collapse bs-navbar-collapse" role="navigation"> - <ul class="nav navbar-nav"> - <li><a href="/index.html" id="home">Home</a></li> - <li><a href="/getting-help.html" id="getting-help">Getting Help</a></li> - <li><a href="/about/integrates.html" id="project-info">Project Information</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="documentation">Documentation <b class="caret"></b></a> - <ul class="dropdown-menu"> - - - <li><a href="/releases/2.0.0-SNAPSHOT/index.html">2.0.0-SNAPSHOT</a></li> - - - - <li><a href="/releases/1.2.0/index.html">1.2.0</a></li> - - - - <li><a href="/releases/1.1.2/index.html">1.1.2</a></li> - - - - <li><a href="/releases/1.1.1/index.html">1.1.1</a></li> - - - - <li><a href="/releases/1.1.0/index.html">1.1.0</a></li> - - - - - - <li><a href="/releases/1.0.6/index.html">1.0.6</a></li> - - - - <li><a href="/releases/1.0.4/index.html">1.0.4</a></li> - - - - - - <li><a href="/releases/1.0.3/index.html">1.0.3</a></li> - - - - - - - - - - <li><a href="/releases/0.10.2/index.html">0.10.2</a></li> - - - - <li><a href="/releases/0.10.1/index.html">0.10.1</a></li> - - - - - - - - <li><a href="/releases/0.9.7/index.html">0.9.7</a></li> - - - - <li><a href="/releases/0.9.6/index.html">0.9.6</a></li> - - - - - - - - - - - - - </ul> - </li> - <li><a href="/talksAndVideos.html">Talks and Slideshows</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="contribute">Community <b class="caret"></b></a> - <ul class="dropdown-menu"> - <li><a href="/contribute/Contributing-to-Storm.html">Contributing</a></li> - <li><a href="/contribute/People.html">People</a></li> - <li><a href="/contribute/BYLAWS.html">ByLaws</a></li> - </ul> - </li> - <li><a href="/2018/02/15/storm120-released.html" id="news">News</a></li> - </ul> - </nav> - </div> -</div> - - - - <div class="container-fluid"> - <h1 class="page-title"></h1> - <div class="row"> - <div class="col-md-12"> - <!-- Documentation --> - -<p class="post-meta"></p> - -<h3 id="basics-of-storm">Basics of Storm</h3> - -<ul> -<li><a href="javadocs/index.html">Javadoc</a></li> -<li><a href="Concepts.html">Concepts</a></li> -<li><a href="Configuration.html">Configuration</a></li> -<li><a href="Guaranteeing-message-processing.html">Guaranteeing message processing</a></li> -<li><a href="Fault-tolerance.html">Fault-tolerance</a></li> -<li><a href="Command-line-client.html">Command line client</a></li> -<li><a href="Understanding-the-parallelism-of-a-Storm-topology.html">Understanding the parallelism of a Storm topology</a></li> -<li><a href="FAQ.html">FAQ</a></li> -</ul> - -<h3 id="trident">Trident</h3> - -<p>Trident is an alternative interface to Storm. It provides exactly-once processing, "transactional" datastore persistence, and a set of common stream analytics operations.</p> - -<ul> -<li><a href="Trident-tutorial.html">Trident Tutorial</a> -- basic concepts and walkthrough</li> -<li><a href="Trident-API-Overview.html">Trident API Overview</a> -- operations for transforming and orchestrating data</li> -<li><a href="Trident-state.html">Trident State</a> -- exactly-once processing and fast, persistent aggregation</li> -<li><a href="Trident-spouts.html">Trident spouts</a> -- transactional and non-transactional data intake</li> -</ul> - -<h3 id="setup-and-deploying">Setup and deploying</h3> - -<ul> -<li><a href="Setting-up-a-Storm-cluster.html">Setting up a Storm cluster</a></li> -<li><a href="Local-mode.html">Local mode</a></li> -<li><a href="Troubleshooting.html">Troubleshooting</a></li> -<li><a href="Running-topologies-on-a-production-cluster.html">Running topologies on a production cluster</a></li> -<li><a href="Maven.html">Building Storm</a> with Maven</li> -</ul> - -<h3 id="intermediate">Intermediate</h3> - -<ul> -<li><a href="Serialization.html">Serialization</a></li> -<li><a href="Common-patterns.html">Common patterns</a></li> -<li><a href="Clojure-DSL.html">Clojure DSL</a></li> -<li><a href="Using-non-JVM-languages-with-Storm.html">Using non-JVM languages with Storm</a></li> -<li><a href="Distributed-RPC.html">Distributed RPC</a></li> -<li><a href="Transactional-topologies.html">Transactional topologies</a></li> -<li><a href="Kestrel-and-Storm.html">Kestrel and Storm</a></li> -<li><a href="Direct-groupings.html">Direct groupings</a></li> -<li><a href="Hooks.html">Hooks</a></li> -<li><a href="Metrics.html">Metrics</a></li> -<li><a href="">Lifecycle of a trident tuple</a></li> -</ul> - -<h3 id="advanced">Advanced</h3> - -<ul> -<li><a href="Defining-a-non-jvm-language-dsl-for-storm.html">Defining a non-JVM language DSL for Storm</a></li> -<li><a href="Multilang-protocol.html">Multilang protocol</a> (how to provide support for another language)</li> -<li><a href="Implementation-docs.html">Implementation docs</a></li> -</ul> - - - - </div> - </div> - </div> -<footer> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Meetups</h5> - <ul class="latest-news"> - - <li><a href="http://www.meetup.com/Apache-Storm-Apache-Kafka/";>Apache Storm & Apache Kafka</a> <span class="small">(Sunnyvale, CA)</span></li> - - <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Apache Storm & Kafka Users</a> <span class="small">(Seattle, WA)</span></li> - - <li><a href="http://www.meetup.com/New-York-City-Storm-User-Group/";>NYC Storm User Group</a> <span class="small">(New York, NY)</span></li> - - <li><a href="http://www.meetup.com/Bay-Area-Stream-Processing";>Bay Area Stream Processing</a> <span class="small">(Emeryville, CA)</span></li> - - <li><a href="http://www.meetup.com/Boston-Storm-Users/";>Boston Realtime Data</a> <span class="small">(Boston, MA)</span></li> - - <li><a href="http://www.meetup.com/storm-london";>London Storm User Group</a> <span class="small">(London, UK)</span></li> - - <!-- <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Seatle, WA</a> <span class="small">(27 Jun 2015)</span></li> --> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>About Storm</h5> - <p>Storm integrates with any queueing system and any database system. Storm's spout abstraction makes it easy to integrate a new queuing system. Likewise, integrating Storm with database systems is easy.</p> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>First Look</h5> - <ul class="footer-list"> - <li><a href="/releases/current/Rationale.html">Rationale</a></li> - <li><a href="/releases/current/Tutorial.html">Tutorial</a></li> - <li><a href="/releases/current/Setting-up-development-environment.html">Setting up development environment</a></li> - <li><a href="/releases/current/Creating-a-new-Storm-project.html">Creating a new Storm project</a></li> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Documentation</h5> - <ul class="footer-list"> - <li><a href="/releases/current/index.html">Index</a></li> - <li><a href="/releases/current/javadocs/index.html">Javadoc</a></li> - <li><a href="/releases/current/FAQ.html">FAQ</a></li> - </ul> - </div> - </div> - </div> - <hr/> - <div class="row"> - <div class="col-md-12"> - <p align="center">Copyright © 2015 <a href="http://www.apache.org";>Apache Software Foundation</a>. All Rights Reserved. - <br>Apache Storm, Apache, the Apache feather logo, and the Apache Storm project logos are trademarks of The Apache Software Foundation. - <br>All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> - </div> - </div> - </div> -</footer> -<!--Footer End--> -<!-- Scroll to top --> -<span class="totop"><a href="#"><i class="fa fa-angle-up"></i></a></span> - -</body> - -</html> -
http://git-wip-us.apache.org/repos/asf/storm-site/blob/ff14ea94/content/releases/0.10.1/FAQ.html ---------------------------------------------------------------------- diff --git a/content/releases/0.10.1/FAQ.html b/content/releases/0.10.1/FAQ.html deleted file mode 100644 index c882d42..0000000 --- a/content/releases/0.10.1/FAQ.html +++ /dev/null @@ -1,382 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <title>FAQ</title> - - <!-- Bootstrap core CSS --> - <link href="/assets/css/bootstrap.min.css" rel="stylesheet"> - <!-- Bootstrap theme --> - <link href="/assets/css/bootstrap-theme.min.css" rel="stylesheet"> - - <!-- Custom styles for this template --> - <link rel="stylesheet" href="http://fortawesome.github.io/Font-Awesome/assets/font-awesome/css/font-awesome.css";> - <link href="/css/style.css" rel="stylesheet"> - <link href="/assets/css/owl.theme.css" rel="stylesheet"> - <link href="/assets/css/owl.carousel.css" rel="stylesheet"> - <script type="text/javascript" src="/assets/js/jquery.min.js"></script> - <script type="text/javascript" src="/assets/js/bootstrap.min.js"></script> - <script type="text/javascript" src="/assets/js/owl.carousel.min.js"></script> - <script type="text/javascript" src="/assets/js/storm.js"></script> - <!-- Just for debugging purposes. Don't actually copy these 2 lines! --> - <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]--> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - - - <body> - <header> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-5"> - <a href="/index.html"><img src="/images/logo.png" class="logo" /></a> - </div> - <div class="col-md-5"> - - <h1>Version: 0.10.1</h1> - - </div> - <div class="col-md-2"> - <a href="/downloads.html" class="btn-std btn-block btn-download">Download</a> - </div> - </div> - </div> -</header> -<!--Header End--> -<!--Navigation Begin--> -<div class="navbar" role="banner"> - <div class="container-fluid"> - <div class="navbar-header"> - <button class="navbar-toggle" type="button" data-toggle="collapse" data-target=".bs-navbar-collapse"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - </div> - <nav class="collapse navbar-collapse bs-navbar-collapse" role="navigation"> - <ul class="nav navbar-nav"> - <li><a href="/index.html" id="home">Home</a></li> - <li><a href="/getting-help.html" id="getting-help">Getting Help</a></li> - <li><a href="/about/integrates.html" id="project-info">Project Information</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="documentation">Documentation <b class="caret"></b></a> - <ul class="dropdown-menu"> - - - <li><a href="/releases/2.0.0-SNAPSHOT/index.html">2.0.0-SNAPSHOT</a></li> - - - - <li><a href="/releases/1.2.0/index.html">1.2.0</a></li> - - - - <li><a href="/releases/1.1.2/index.html">1.1.2</a></li> - - - - <li><a href="/releases/1.1.1/index.html">1.1.1</a></li> - - - - <li><a href="/releases/1.1.0/index.html">1.1.0</a></li> - - - - - - <li><a href="/releases/1.0.6/index.html">1.0.6</a></li> - - - - <li><a href="/releases/1.0.4/index.html">1.0.4</a></li> - - - - - - <li><a href="/releases/1.0.3/index.html">1.0.3</a></li> - - - - - - - - - - <li><a href="/releases/0.10.2/index.html">0.10.2</a></li> - - - - <li><a href="/releases/0.10.1/index.html">0.10.1</a></li> - - - - - - - - <li><a href="/releases/0.9.7/index.html">0.9.7</a></li> - - - - <li><a href="/releases/0.9.6/index.html">0.9.6</a></li> - - - - - - - - - - - - - </ul> - </li> - <li><a href="/talksAndVideos.html">Talks and Slideshows</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="contribute">Community <b class="caret"></b></a> - <ul class="dropdown-menu"> - <li><a href="/contribute/Contributing-to-Storm.html">Contributing</a></li> - <li><a href="/contribute/People.html">People</a></li> - <li><a href="/contribute/BYLAWS.html">ByLaws</a></li> - </ul> - </li> - <li><a href="/2018/02/15/storm120-released.html" id="news">News</a></li> - </ul> - </nav> - </div> -</div> - - - - <div class="container-fluid"> - <h1 class="page-title">FAQ</h1> - <div class="row"> - <div class="col-md-12"> - <!-- Documentation --> - -<p class="post-meta"></p> - -<h2 id="best-practices">Best Practices</h2> - -<h3 id="what-rules-of-thumb-can-you-give-me-for-configuring-storm-trident">What rules of thumb can you give me for configuring Storm+Trident?</h3> - -<ul> -<li>number of workers a multiple of number of machines; parallelism a multiple of number of workers; number of kafka partitions a multiple of number of spout parallelism</li> -<li>Use one worker per topology per machine</li> -<li>Start with fewer, larger aggregators, one per machine with workers on it</li> -<li>Use the isolation scheduler</li> -<li>Use one acker per worker -- 0.9 makes that the default, but earlier versions do not.</li> -<li>enable GC logging; you should see very few major GCs if things are in reasonable shape.</li> -<li>set the trident batch millis to about 50% of your typical end-to-end latency.</li> -<li>Start with a max spout pending that is for sure too small -- one for trident, or the number of executors for storm -- and increase it until you stop seeing changes in the flow. You'll probably end up with something near <code>2*(throughput in recs/sec)*(end-to-end latency)</code> (2x the Little's law capacity).</li> -</ul> - -<h3 id="what-are-some-of-the-best-ways-to-get-a-worker-to-mysteriously-and-bafflingly-die">What are some of the best ways to get a worker to mysteriously and bafflingly die?</h3> - -<ul> -<li>Do you have write access to the log directory</li> -<li>Are you blowing out your heap?</li> -<li>Are all the right libraries installed on all of the workers?</li> -<li>Is the zookeeper hostname still set to localhost?</li> -<li>Did you supply a correct, unique hostname -- one that resolves back to the machine -- to each worker, and put it in the storm conf file?</li> -<li>Have you opened firewall/securitygroup permissions <em>bidirectionally</em> among a) all the workers, b) the storm master, c) zookeeper? Also, from the workers to any kafka/kestrel/database/etc that your topology accesses? Use netcat to poke the appropriate ports and be sure. </li> -</ul> - -<h3 id="halp-i-cannot-see">Halp! I cannot see:</h3> - -<ul> -<li><strong>my logs</strong> Logs by default go to $STORM_HOME/logs. Check that you have write permissions to that directory. They are configured in log4j2/{cluster, worker}.xml.</li> -<li><strong>final JVM settings</strong> Add the <code>-XX+PrintFlagsFinal</code> commandline option in the childopts (see the conf file)</li> -<li><strong>final Java system properties</strong> Add <code>Properties props = System.getProperties(); props.list(System.out);</code> near where you build your topology.</li> -</ul> - -<h3 id="how-many-workers-should-i-use">How many Workers should I use?</h3> - -<p>The total number of workers is set by the supervisors -- there's some number of JVM slots each supervisor will superintend. The thing you set on the topology is how many worker slots it will try to claim.</p> - -<p>There's no great reason to use more than one worker per topology per machine.</p> - -<p>With one topology running on three 8-core nodes, and parallelism hint 24, each bolt gets 8 executors per machine, i.e. one for each core. There are three big benefits to running three workers (with 8 assigned executors each) compare to running say 24 workers (one assigned executor each).</p> - -<p>First, data that is repartitioned (shuffles or group-bys) to executors in the same worker will not have to hit the transfer buffer. Instead, tuples are deposited directly from send to receive buffer. That's a big win. By contrast, if the destination executor were on the same machine in a different worker, it would have to go send -> worker transfer -> local socket -> worker recv -> exec recv buffer. It doesn't hit the network card, but it's not as big a win as when executors are in the same worker.</p> - -<p>Second, you're typically better off with three aggregators having very large backing cache than having twenty-four aggregators having small backing caches. This reduces the effect of skew, and improves LRU efficiency.</p> - -<p>Lastly, fewer workers reduces control flow chatter.</p> - -<h2 id="topology">Topology</h2> - -<h3 id="can-a-trident-topology-have-multiple-streams">Can a Trident topology have Multiple Streams?</h3> - -<blockquote> -<p>Can a Trident Topology work like a workflow with conditional paths (if-else)? e.g. A Spout (S1) connects to a bolt (B0) which based on certain values in the incoming tuple routes them to either bolt (B1) or bolt (B2) but not both.</p> -</blockquote> - -<p>A Trident "each" operator returns a Stream object, which you can store in a variable. You can then run multiple eaches on the same Stream to split it, e.g.: </p> -<div class="highlight"><pre><code class="language-" data-lang=""> Stream s = topology.each(...).groupBy(...).aggregate(...) - Stream branch1 = s.each(..., FilterA) - Stream branch2 = s.each(..., FilterB) -</code></pre></div> -<p>You can join streams with join, merge or multiReduce.</p> - -<p>At time of writing, you can't emit to multiple output streams from Trident -- see <a href="https://issues.apache.org/jira/browse/STORM-68";>STORM-68</a></p> - -<h3 id="why-am-i-getting-a-notserializableexception-illegalstateexception-when-my-topology-is-being-started-up">Why am I getting a NotSerializableException/IllegalStateException when my topology is being started up?</h3> - -<p>Within the Storm lifecycle, the topology is instantiated and then serialized to byte format to be stored in ZooKeeper, prior to the topology being executed. Within this step, if a spout or bolt within the topology has an initialized unserializable property, serialization will fail. If there is a need for a field that is unserializable, initialize it within the bolt or spout's prepare method, which is run after the topology is delivered to the worker.</p> - -<h2 id="spouts">Spouts</h2> - -<h3 id="what-is-a-coordinator-and-why-are-there-several">What is a coordinator, and why are there several?</h3> - -<p>A trident-spout is actually run within a storm <em>bolt</em>. The storm-spout of a trident topology is the MasterBatchCoordinator -- it coordinates trident batches and is the same no matter what spouts you use. A batch is born when the MBC dispenses a seed tuple to each of the spout-coordinators. The spout-coordinator bolts know how your particular spouts should cooperate -- so in the kafka case, it's what helps figure out what partition and offset range each spout should pull from.</p> - -<h3 id="what-can-i-store-into-the-spouts-metadata-record">What can I store into the spout's metadata record?</h3> - -<p>You should only store static data, and as little of it as possible, into the metadata record (note: maybe you <em>can</em> store more interesting things; you shouldn't, though)</p> - -<h3 id="how-often-is-the-emitpartitionbatchnew-function-called">How often is the 'emitPartitionBatchNew' function called?</h3> - -<p>Since the MBC is the actual spout, all the tuples in a batch are just members of its tupletree. That means storm's "max spout pending" config effectively defines the number of concurrent batches trident runs. The MBC emits a new batch if it has fewer than max-spending tuples pending and if at least one <a href="http://github.com/apache/storm/blob/v0.10.1/conf/defaults.yaml#L115";>trident batch interval</a>'s worth of seconds has passed since the last batch.</p> - -<h3 id="if-nothing-was-emitted-does-trident-slow-down-the-calls">If nothing was emitted does Trident slow down the calls?</h3> - -<p>Yes, there's a pluggable "spout wait strategy"; the default is to sleep for a <a href="http://github.com/apache/storm/blob/v0.10.1/conf/defaults.yaml#L110";>configurable amount of time</a></p> - -<h3 id="ok-then-what-is-the-trident-batch-interval-for">OK, then what is the trident batch interval for?</h3> - -<p>You know how computers of the 486 era had a <a href="http://en.wikipedia.org/wiki/Turbo_button";>turbo button</a> on them? It's like that. </p> - -<p>Actually, it has two practical uses. One is to throttle spouts that poll a remote source without throttling processing. For example, we have a spout that looks in a given S3 bucket for a new batch-uploaded file to read, linebreak and emit. We don't want it hitting S3 more than every few seconds: files don't show up more than once every few minutes, and a batch takes a few seconds to process.</p> - -<p>The other is to limit overpressure on the internal queues during startup or under a heavy burst load -- if the spouts spring to life and suddenly jam ten batches' worth of records into the system, you could have a mass of less-urgent tuples from batch 7 clog up the transfer buffer and prevent the $commit tuple from batch 3 to get through (or even just the regular old tuples from batch 3). What we do is set the trident batch interval to about half the typical end-to-end processing latency -- if it takes 600ms to process a batch, it's OK to only kick off a batch every 300ms.</p> - -<p>Note that this is a cap, not an additional delay -- with a period of 300ms, if your batch takes 258ms Trident will only delay an additional 42ms.</p> - -<h3 id="how-do-you-set-the-batch-size">How do you set the batch size?</h3> - -<p>Trident doesn't place its own limits on the batch count. In the case of the Kafka spout, the max fetch bytes size divided by the average record size defines an effective records per subbatch partition.</p> - -<h3 id="how-do-i-resize-a-batch">How do I resize a batch?</h3> - -<p>The trident batch is a somewhat overloaded facility. Together with the number of partitions, the batch size is constrained by or serves to define</p> - -<ol> -<li>the unit of transactional safety (tuples at risk vs time)</li> -<li>per partition, an effective windowing mechanism for windowed stream analytics</li> -<li>per partition, the number of simultaneous queries that will be made by a partitionQuery, partitionPersist, etc;</li> -<li>per partition, the number of records convenient for the spout to dispatch at the same time;</li> -</ol> - -<p>You can't change the overall batch size once generated, but you can change the number of partitions -- do a shuffle and then change the parallelism hint</p> - -<h2 id="time-series">Time Series</h2> - -<h3 id="how-do-i-aggregate-events-by-time">How do I aggregate events by time?</h3> - -<p>If you have records with an immutable timestamp, and you would like to count, average or otherwise aggregate them into discrete time buckets, Trident is an excellent and scalable solution.</p> - -<p>Write an <code>Each</code> function that turns the timestamp into a time bucket: if the bucket size was "by hour", then the timestamp <code>2013-08-08 12:34:56</code> would be mapped to the <code>2013-08-08 12:00:00</code> time bucket, and so would everything else in the twelve o'clock hour. Then group on that timebucket and use a grouped persistentAggregate. The persistentAggregate uses a local cacheMap backed by a data store. Groups with many records require very few reads from the data store, and use efficient bulk reads and writes; as long as your data feed is relatively prompt Trident will make very efficient use of memory and network. Even if a server drops off line for a day, then delivers that full day's worth of data in a rush, the old results will be calmly retrieved and updated -- and without interfering with calculating the current results.</p> - -<h3 id="how-can-i-know-that-all-records-for-a-time-bucket-have-been-received">How can I know that all records for a time bucket have been received?</h3> - -<p>You cannot know that all events are collected -- this is an epistemological challenge, not a distributed systems challenge. You can:</p> - -<ul> -<li>Set a time limit using domain knowledge</li> -<li>Introduce a <em>punctuation</em>: a record known to come after all records in the given time bucket. Trident uses this scheme to know when a batch is complete. If you for instance receive records from a set of sensors, each in order for that sensor, then once all sensors have sent you a 3:02:xx or later timestamp lets you know you can commit. </li> -<li>When possible, make your process incremental: each value that comes in makes the answer more an more true. A Trident ReducerAggregator is an operator that takes a prior result and a set of new records and returns a new result. This lets the result be cached and serialized to a datastore; if a server drops off line for a day and then comes back with a full day's worth of data in a rush, the old results will be calmly retrieved and updated.</li> -<li>Lambda architecture: Record all events into an archival store (S3, HBase, HDFS) on receipt. in the fast layer, once the time window is clear, process the bucket to get an actionable answer, and ignore everything older than the time window. Periodically run a global aggregation to calculate a "correct" answer.</li> -</ul> - - - - </div> - </div> - </div> -<footer> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Meetups</h5> - <ul class="latest-news"> - - <li><a href="http://www.meetup.com/Apache-Storm-Apache-Kafka/";>Apache Storm & Apache Kafka</a> <span class="small">(Sunnyvale, CA)</span></li> - - <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Apache Storm & Kafka Users</a> <span class="small">(Seattle, WA)</span></li> - - <li><a href="http://www.meetup.com/New-York-City-Storm-User-Group/";>NYC Storm User Group</a> <span class="small">(New York, NY)</span></li> - - <li><a href="http://www.meetup.com/Bay-Area-Stream-Processing";>Bay Area Stream Processing</a> <span class="small">(Emeryville, CA)</span></li> - - <li><a href="http://www.meetup.com/Boston-Storm-Users/";>Boston Realtime Data</a> <span class="small">(Boston, MA)</span></li> - - <li><a href="http://www.meetup.com/storm-london";>London Storm User Group</a> <span class="small">(London, UK)</span></li> - - <!-- <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Seatle, WA</a> <span class="small">(27 Jun 2015)</span></li> --> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>About Storm</h5> - <p>Storm integrates with any queueing system and any database system. Storm's spout abstraction makes it easy to integrate a new queuing system. Likewise, integrating Storm with database systems is easy.</p> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>First Look</h5> - <ul class="footer-list"> - <li><a href="/releases/current/Rationale.html">Rationale</a></li> - <li><a href="/releases/current/Tutorial.html">Tutorial</a></li> - <li><a href="/releases/current/Setting-up-development-environment.html">Setting up development environment</a></li> - <li><a href="/releases/current/Creating-a-new-Storm-project.html">Creating a new Storm project</a></li> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Documentation</h5> - <ul class="footer-list"> - <li><a href="/releases/current/index.html">Index</a></li> - <li><a href="/releases/current/javadocs/index.html">Javadoc</a></li> - <li><a href="/releases/current/FAQ.html">FAQ</a></li> - </ul> - </div> - </div> - </div> - <hr/> - <div class="row"> - <div class="col-md-12"> - <p align="center">Copyright © 2015 <a href="http://www.apache.org";>Apache Software Foundation</a>. All Rights Reserved. - <br>Apache Storm, Apache, the Apache feather logo, and the Apache Storm project logos are trademarks of The Apache Software Foundation. - <br>All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> - </div> - </div> - </div> -</footer> -<!--Footer End--> -<!-- Scroll to top --> -<span class="totop"><a href="#"><i class="fa fa-angle-up"></i></a></span> - -</body> - -</html> - http://git-wip-us.apache.org/repos/asf/storm-site/blob/ff14ea94/content/releases/0.10.1/Fault-tolerance.html ---------------------------------------------------------------------- diff --git a/content/releases/0.10.1/Fault-tolerance.html b/content/releases/0.10.1/Fault-tolerance.html deleted file mode 100644 index 69041cd..0000000 --- a/content/releases/0.10.1/Fault-tolerance.html +++ /dev/null @@ -1,275 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <title>Apache Storm</title> - - <!-- Bootstrap core CSS --> - <link href="/assets/css/bootstrap.min.css" rel="stylesheet"> - <!-- Bootstrap theme --> - <link href="/assets/css/bootstrap-theme.min.css" rel="stylesheet"> - - <!-- Custom styles for this template --> - <link rel="stylesheet" href="http://fortawesome.github.io/Font-Awesome/assets/font-awesome/css/font-awesome.css";> - <link href="/css/style.css" rel="stylesheet"> - <link href="/assets/css/owl.theme.css" rel="stylesheet"> - <link href="/assets/css/owl.carousel.css" rel="stylesheet"> - <script type="text/javascript" src="/assets/js/jquery.min.js"></script> - <script type="text/javascript" src="/assets/js/bootstrap.min.js"></script> - <script type="text/javascript" src="/assets/js/owl.carousel.min.js"></script> - <script type="text/javascript" src="/assets/js/storm.js"></script> - <!-- Just for debugging purposes. Don't actually copy these 2 lines! --> - <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]--> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - - - <body> - <header> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-5"> - <a href="/index.html"><img src="/images/logo.png" class="logo" /></a> - </div> - <div class="col-md-5"> - - <h1>Version: 0.10.1</h1> - - </div> - <div class="col-md-2"> - <a href="/downloads.html" class="btn-std btn-block btn-download">Download</a> - </div> - </div> - </div> -</header> -<!--Header End--> -<!--Navigation Begin--> -<div class="navbar" role="banner"> - <div class="container-fluid"> - <div class="navbar-header"> - <button class="navbar-toggle" type="button" data-toggle="collapse" data-target=".bs-navbar-collapse"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - </div> - <nav class="collapse navbar-collapse bs-navbar-collapse" role="navigation"> - <ul class="nav navbar-nav"> - <li><a href="/index.html" id="home">Home</a></li> - <li><a href="/getting-help.html" id="getting-help">Getting Help</a></li> - <li><a href="/about/integrates.html" id="project-info">Project Information</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="documentation">Documentation <b class="caret"></b></a> - <ul class="dropdown-menu"> - - - <li><a href="/releases/2.0.0-SNAPSHOT/index.html">2.0.0-SNAPSHOT</a></li> - - - - <li><a href="/releases/1.2.0/index.html">1.2.0</a></li> - - - - <li><a href="/releases/1.1.2/index.html">1.1.2</a></li> - - - - <li><a href="/releases/1.1.1/index.html">1.1.1</a></li> - - - - <li><a href="/releases/1.1.0/index.html">1.1.0</a></li> - - - - - - <li><a href="/releases/1.0.6/index.html">1.0.6</a></li> - - - - <li><a href="/releases/1.0.4/index.html">1.0.4</a></li> - - - - - - <li><a href="/releases/1.0.3/index.html">1.0.3</a></li> - - - - - - - - - - <li><a href="/releases/0.10.2/index.html">0.10.2</a></li> - - - - <li><a href="/releases/0.10.1/index.html">0.10.1</a></li> - - - - - - - - <li><a href="/releases/0.9.7/index.html">0.9.7</a></li> - - - - <li><a href="/releases/0.9.6/index.html">0.9.6</a></li> - - - - - - - - - - - - - </ul> - </li> - <li><a href="/talksAndVideos.html">Talks and Slideshows</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="contribute">Community <b class="caret"></b></a> - <ul class="dropdown-menu"> - <li><a href="/contribute/Contributing-to-Storm.html">Contributing</a></li> - <li><a href="/contribute/People.html">People</a></li> - <li><a href="/contribute/BYLAWS.html">ByLaws</a></li> - </ul> - </li> - <li><a href="/2018/02/15/storm120-released.html" id="news">News</a></li> - </ul> - </nav> - </div> -</div> - - - - <div class="container-fluid"> - <h1 class="page-title"></h1> - <div class="row"> - <div class="col-md-12"> - <!-- Documentation --> - -<p class="post-meta"></p> - -<p>This page explains the design details of Storm that make it a fault-tolerant system.</p> - -<h2 id="what-happens-when-a-worker-dies">What happens when a worker dies?</h2> - -<p>When a worker dies, the supervisor will restart it. If it continuously fails on startup and is unable to heartbeat to Nimbus, Nimbus will reassign the worker to another machine.</p> - -<h2 id="what-happens-when-a-node-dies">What happens when a node dies?</h2> - -<p>The tasks assigned to that machine will time-out and Nimbus will reassign those tasks to other machines.</p> - -<h2 id="what-happens-when-nimbus-or-supervisor-daemons-die">What happens when Nimbus or Supervisor daemons die?</h2> - -<p>The Nimbus and Supervisor daemons are designed to be fail-fast (process self-destructs whenever any unexpected situation is encountered) and stateless (all state is kept in Zookeeper or on disk). As described in <a href="Setting-up-a-Storm-cluster.html">Setting up a Storm cluster</a>, the Nimbus and Supervisor daemons must be run under supervision using a tool like daemontools or monit. So if the Nimbus or Supervisor daemons die, they restart like nothing happened.</p> - -<p>Most notably, no worker processes are affected by the death of Nimbus or the Supervisors. This is in contrast to Hadoop, where if the JobTracker dies, all the running jobs are lost. </p> - -<h2 id="is-nimbus-a-single-point-of-failure">Is Nimbus a single point of failure?</h2> - -<p>If you lose the Nimbus node, the workers will still continue to function. Additionally, supervisors will continue to restart workers if they die. However, without Nimbus, workers won't be reassigned to other machines when necessary (like if you lose a worker machine). </p> - -<p>So the answer is that Nimbus is "sort of" a SPOF. In practice, it's not a big deal since nothing catastrophic happens when the Nimbus daemon dies. There are plans to make Nimbus highly available in the future.</p> - -<h2 id="how-does-storm-guarantee-data-processing">How does Storm guarantee data processing?</h2> - -<p>Storm provides mechanisms to guarantee data processing even if nodes die or messages are lost. See <a href="Guaranteeing-message-processing.html">Guaranteeing message processing</a> for the details.</p> - - - - </div> - </div> - </div> -<footer> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Meetups</h5> - <ul class="latest-news"> - - <li><a href="http://www.meetup.com/Apache-Storm-Apache-Kafka/";>Apache Storm & Apache Kafka</a> <span class="small">(Sunnyvale, CA)</span></li> - - <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Apache Storm & Kafka Users</a> <span class="small">(Seattle, WA)</span></li> - - <li><a href="http://www.meetup.com/New-York-City-Storm-User-Group/";>NYC Storm User Group</a> <span class="small">(New York, NY)</span></li> - - <li><a href="http://www.meetup.com/Bay-Area-Stream-Processing";>Bay Area Stream Processing</a> <span class="small">(Emeryville, CA)</span></li> - - <li><a href="http://www.meetup.com/Boston-Storm-Users/";>Boston Realtime Data</a> <span class="small">(Boston, MA)</span></li> - - <li><a href="http://www.meetup.com/storm-london";>London Storm User Group</a> <span class="small">(London, UK)</span></li> - - <!-- <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Seatle, WA</a> <span class="small">(27 Jun 2015)</span></li> --> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>About Storm</h5> - <p>Storm integrates with any queueing system and any database system. Storm's spout abstraction makes it easy to integrate a new queuing system. Likewise, integrating Storm with database systems is easy.</p> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>First Look</h5> - <ul class="footer-list"> - <li><a href="/releases/current/Rationale.html">Rationale</a></li> - <li><a href="/releases/current/Tutorial.html">Tutorial</a></li> - <li><a href="/releases/current/Setting-up-development-environment.html">Setting up development environment</a></li> - <li><a href="/releases/current/Creating-a-new-Storm-project.html">Creating a new Storm project</a></li> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Documentation</h5> - <ul class="footer-list"> - <li><a href="/releases/current/index.html">Index</a></li> - <li><a href="/releases/current/javadocs/index.html">Javadoc</a></li> - <li><a href="/releases/current/FAQ.html">FAQ</a></li> - </ul> - </div> - </div> - </div> - <hr/> - <div class="row"> - <div class="col-md-12"> - <p align="center">Copyright © 2015 <a href="http://www.apache.org";>Apache Software Foundation</a>. All Rights Reserved. - <br>Apache Storm, Apache, the Apache feather logo, and the Apache Storm project logos are trademarks of The Apache Software Foundation. - <br>All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> - </div> - </div> - </div> -</footer> -<!--Footer End--> -<!-- Scroll to top --> -<span class="totop"><a href="#"><i class="fa fa-angle-up"></i></a></span> - -</body> - -</html> - http://git-wip-us.apache.org/repos/asf/storm-site/blob/ff14ea94/content/releases/0.10.1/Guaranteeing-message-processing.html ---------------------------------------------------------------------- diff --git a/content/releases/0.10.1/Guaranteeing-message-processing.html b/content/releases/0.10.1/Guaranteeing-message-processing.html deleted file mode 100644 index b3eb19f..0000000 --- a/content/releases/0.10.1/Guaranteeing-message-processing.html +++ /dev/null @@ -1,407 +0,0 @@ -<!DOCTYPE html> -<html> - <head> - <meta charset="utf-8"> - <meta http-equiv="X-UA-Compatible" content="IE=edge"> - <meta name="viewport" content="width=device-width, initial-scale=1"> - - <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon"> - <link rel="icon" href="/favicon.ico" type="image/x-icon"> - - <title>Guaranteeing Message Processing</title> - - <!-- Bootstrap core CSS --> - <link href="/assets/css/bootstrap.min.css" rel="stylesheet"> - <!-- Bootstrap theme --> - <link href="/assets/css/bootstrap-theme.min.css" rel="stylesheet"> - - <!-- Custom styles for this template --> - <link rel="stylesheet" href="http://fortawesome.github.io/Font-Awesome/assets/font-awesome/css/font-awesome.css";> - <link href="/css/style.css" rel="stylesheet"> - <link href="/assets/css/owl.theme.css" rel="stylesheet"> - <link href="/assets/css/owl.carousel.css" rel="stylesheet"> - <script type="text/javascript" src="/assets/js/jquery.min.js"></script> - <script type="text/javascript" src="/assets/js/bootstrap.min.js"></script> - <script type="text/javascript" src="/assets/js/owl.carousel.min.js"></script> - <script type="text/javascript" src="/assets/js/storm.js"></script> - <!-- Just for debugging purposes. Don't actually copy these 2 lines! --> - <!--[if lt IE 9]><script src="../../assets/js/ie8-responsive-file-warning.js"></script><![endif]--> - - <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries --> - <!--[if lt IE 9]> - <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js";></script> - <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js";></script> - <![endif]--> - </head> - - - <body> - <header> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-5"> - <a href="/index.html"><img src="/images/logo.png" class="logo" /></a> - </div> - <div class="col-md-5"> - - <h1>Version: 0.10.1</h1> - - </div> - <div class="col-md-2"> - <a href="/downloads.html" class="btn-std btn-block btn-download">Download</a> - </div> - </div> - </div> -</header> -<!--Header End--> -<!--Navigation Begin--> -<div class="navbar" role="banner"> - <div class="container-fluid"> - <div class="navbar-header"> - <button class="navbar-toggle" type="button" data-toggle="collapse" data-target=".bs-navbar-collapse"> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - <span class="icon-bar"></span> - </button> - </div> - <nav class="collapse navbar-collapse bs-navbar-collapse" role="navigation"> - <ul class="nav navbar-nav"> - <li><a href="/index.html" id="home">Home</a></li> - <li><a href="/getting-help.html" id="getting-help">Getting Help</a></li> - <li><a href="/about/integrates.html" id="project-info">Project Information</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="documentation">Documentation <b class="caret"></b></a> - <ul class="dropdown-menu"> - - - <li><a href="/releases/2.0.0-SNAPSHOT/index.html">2.0.0-SNAPSHOT</a></li> - - - - <li><a href="/releases/1.2.0/index.html">1.2.0</a></li> - - - - <li><a href="/releases/1.1.2/index.html">1.1.2</a></li> - - - - <li><a href="/releases/1.1.1/index.html">1.1.1</a></li> - - - - <li><a href="/releases/1.1.0/index.html">1.1.0</a></li> - - - - - - <li><a href="/releases/1.0.6/index.html">1.0.6</a></li> - - - - <li><a href="/releases/1.0.4/index.html">1.0.4</a></li> - - - - - - <li><a href="/releases/1.0.3/index.html">1.0.3</a></li> - - - - - - - - - - <li><a href="/releases/0.10.2/index.html">0.10.2</a></li> - - - - <li><a href="/releases/0.10.1/index.html">0.10.1</a></li> - - - - - - - - <li><a href="/releases/0.9.7/index.html">0.9.7</a></li> - - - - <li><a href="/releases/0.9.6/index.html">0.9.6</a></li> - - - - - - - - - - - - - </ul> - </li> - <li><a href="/talksAndVideos.html">Talks and Slideshows</a></li> - <li class="dropdown"> - <a href="#" class="dropdown-toggle" data-toggle="dropdown" id="contribute">Community <b class="caret"></b></a> - <ul class="dropdown-menu"> - <li><a href="/contribute/Contributing-to-Storm.html">Contributing</a></li> - <li><a href="/contribute/People.html">People</a></li> - <li><a href="/contribute/BYLAWS.html">ByLaws</a></li> - </ul> - </li> - <li><a href="/2018/02/15/storm120-released.html" id="news">News</a></li> - </ul> - </nav> - </div> -</div> - - - - <div class="container-fluid"> - <h1 class="page-title">Guaranteeing Message Processing</h1> - <div class="row"> - <div class="col-md-12"> - <!-- Documentation --> - -<p class="post-meta"></p> - -<p>Storm offers several different levels of guaranteed message processing, includeing best effort, at least once, and exactly once through <a href="Trident-tutorial.html">Trident</a>. -This page describes how Storm can guarantee at least once processing.</p> - -<h3 id="what-does-it-mean-for-a-message-to-be-fully-processed">What does it mean for a message to be "fully processed"?</h3> - -<p>A tuple coming off a spout can trigger thousands of tuples to be created based on it. Consider, for example, the streaming word count topology:</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">TopologyBuilder</span> <span class="n">builder</span> <span class="o">=</span> <span class="k">new</span> <span class="n">TopologyBuilder</span><span class="o">();</span> -<span class="n">builder</span><span class="o">.</span><span class="na">setSpout</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">,</span> <span class="k">new</span> <span class="n">KestrelSpout</span><span class="o">(</span><span class="s">"kestrel.backtype.com"</span><span class="o">,</span> - <span class="mi">22133</span><span class="o">,</span> - <span class="s">"sentence_queue"</span><span class="o">,</span> - <span class="k">new</span> <span class="nf">StringScheme</span><span class="o">()));</span> -<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="n">SplitSentence</span><span class="o">(),</span> <span class="mi">10</span><span class="o">)</span> - <span class="o">.</span><span class="na">shuffleGrouping</span><span class="o">(</span><span class="s">"sentences"</span><span class="o">);</span> -<span class="n">builder</span><span class="o">.</span><span class="na">setBolt</span><span class="o">(</span><span class="s">"count"</span><span class="o">,</span> <span class="k">new</span> <span class="n">WordCount</span><span class="o">(),</span> <span class="mi">20</span><span class="o">)</span> - <span class="o">.</span><span class="na">fieldsGrouping</span><span class="o">(</span><span class="s">"split"</span><span class="o">,</span> <span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span> -</code></pre></div> -<p>This topology reads sentences off of a Kestrel queue, splits the sentences into its constituent words, and then emits for each word the number of times it has seen that word before. A tuple coming off the spout triggers many tuples being created based on it: a tuple for each word in the sentence and a tuple for the updated count for each word. The tree of messages looks something like this:</p> - -<p><img src="images/tuple_tree.png" alt="Tuple tree"></p> - -<p>Storm considers a tuple coming off a spout "fully processed" when the tuple tree has been exhausted and every message in the tree has been processed. A tuple is considered failed when its tree of messages fails to be fully processed within a specified timeout. This timeout can be configured on a topology-specific basis using the <a href="javadocs/backtype/storm/Config.html#TOPOLOGY_MESSAGE_TIMEOUT_SECS">Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS</a> configuration and defaults to 30 seconds.</p> - -<h3 id="what-happens-if-a-message-is-fully-processed-or-fails-to-be-fully-processed">What happens if a message is fully processed or fails to be fully processed?</h3> - -<p>To understand this question, let's take a look at the lifecycle of a tuple coming off of a spout. For reference, here is the interface that spouts implement (see the <a href="javadocs/backtype/storm/spout/ISpout.html">Javadoc</a> for more information):</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">interface</span> <span class="nc">ISpout</span> <span class="kd">extends</span> <span class="n">Serializable</span> <span class="o">{</span> - <span class="kt">void</span> <span class="nf">open</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">SpoutOutputCollector</span> <span class="n">collector</span><span class="o">);</span> - <span class="kt">void</span> <span class="nf">close</span><span class="o">();</span> - <span class="kt">void</span> <span class="nf">nextTuple</span><span class="o">();</span> - <span class="kt">void</span> <span class="nf">ack</span><span class="o">(</span><span class="n">Object</span> <span class="n">msgId</span><span class="o">);</span> - <span class="kt">void</span> <span class="nf">fail</span><span class="o">(</span><span class="n">Object</span> <span class="n">msgId</span><span class="o">);</span> -<span class="o">}</span> -</code></pre></div> -<p>First, Storm requests a tuple from the <code>Spout</code> by calling the <code>nextTuple</code> method on the <code>Spout</code>. The <code>Spout</code> uses the <code>SpoutOutputCollector</code> provided in the <code>open</code> method to emit a tuple to one of its output streams. When emitting a tuple, the <code>Spout</code> provides a "message id" that will be used to identify the tuple later. For example, the <code>KestrelSpout</code> reads a message off of the kestrel queue and emits as the "message id" the id provided by Kestrel for the message. Emitting a message to the <code>SpoutOutputCollector</code> looks like this:</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="s">"field1"</span><span class="o">,</span> <span class="s">"field2"</span><span class="o">,</span> <span class="mi">3</span><span class="o">)</span> <span class="o">,</span> <span class="n">msgId</span><span class="o">);</span> -</code></pre></div> -<p>Next, the tuple gets sent to consuming bolts and Storm takes care of tracking the tree of messages that is created. If Storm detects that a tuple is fully processed, Storm will call the <code>ack</code> method on the originating <code>Spout</code> task with the message id that the <code>Spout</code> provided to Storm. Likewise, if the tuple times-out Storm will call the <code>fail</code> method on the <code>Spout</code>. Note that a tuple will be acked or failed by the exact same <code>Spout</code> task that created it. So if a <code>Spout</code> is executing as many tasks across the cluster, a tuple won't be acked or failed by a different task than the one that created it.</p> - -<p>Let's use <code>KestrelSpout</code> again to see what a <code>Spout</code> needs to do to guarantee message processing. When <code>KestrelSpout</code> takes a message off the Kestrel queue, it "opens" the message. This means the message is not actually taken off the queue yet, but instead placed in a "pending" state waiting for acknowledgement that the message is completed. While in the pending state, a message will not be sent to other consumers of the queue. Additionally, if a client disconnects all pending messages for that client are put back on the queue. When a message is opened, Kestrel provides the client with the data for the message as well as a unique id for the message. The <code>KestrelSpout</code> uses that exact id as the "message id" for the tuple when emitting the tuple to the <code>SpoutOutputCollector</code>. Sometime later on, when <code>ack</code> or <code>fail</code> are called on the <code>KestrelSpout</code>, the <code>Kes trelSpout</code> sends an ack or fail message to Kestrel with the message id to take the message off the queue or have it put back on.</p> - -<h3 id="what-is-storms-reliability-api">What is Storm's reliability API?</h3> - -<p>There's two things you have to do as a user to benefit from Storm's reliability capabilities. First, you need to tell Storm whenever you're creating a new link in the tree of tuples. Second, you need to tell Storm when you have finished processing an individual tuple. By doing both these things, Storm can detect when the tree of tuples is fully processed and can ack or fail the spout tuple appropriately. Storm's API provides a concise way of doing both of these tasks. </p> - -<p>Specifying a link in the tuple tree is called <em>anchoring</em>. Anchoring is done at the same time you emit a new tuple. Let's use the following bolt as an example. This bolt splits a tuple containing a sentence into a tuple for each word:</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SplitSentence</span> <span class="kd">extends</span> <span class="n">BaseRichBolt</span> <span class="o">{</span> - <span class="n">OutputCollector</span> <span class="n">_collector</span><span class="o">;</span> - - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">prepare</span><span class="o">(</span><span class="n">Map</span> <span class="n">conf</span><span class="o">,</span> <span class="n">TopologyContext</span> <span class="n">context</span><span class="o">,</span> <span class="n">OutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span> - <span class="n">_collector</span> <span class="o">=</span> <span class="n">collector</span><span class="o">;</span> - <span class="o">}</span> - - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">)</span> <span class="o">{</span> - <span class="n">String</span> <span class="n">sentence</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span> - <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span> - <span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">tuple</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span> - <span class="o">}</span> - <span class="n">_collector</span><span class="o">.</span><span class="na">ack</span><span class="o">(</span><span class="n">tuple</span><span class="o">);</span> - <span class="o">}</span> - - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span> - <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span> - <span class="o">}</span> - <span class="o">}</span> -</code></pre></div> -<p>Each word tuple is <em>anchored</em> by specifying the input tuple as the first argument to <code>emit</code>. Since the word tuple is anchored, the spout tuple at the root of the tree will be replayed later on if the word tuple failed to be processed downstream. In contrast, let's look at what happens if the word tuple is emitted like this:</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span> -</code></pre></div> -<p>Emitting the word tuple this way causes it to be <em>unanchored</em>. If the tuple fails be processed downstream, the root tuple will not be replayed. Depending on the fault-tolerance guarantees you need in your topology, sometimes it's appropriate to emit an unanchored tuple.</p> - -<p>An output tuple can be anchored to more than one input tuple. This is useful when doing streaming joins or aggregations. A multi-anchored tuple failing to be processed will cause multiple tuples to be replayed from the spouts. Multi-anchoring is done by specifying a list of tuples rather than just a single tuple. For example:</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="n">List</span><span class="o"><</span><span class="n">Tuple</span><span class="o">></span> <span class="n">anchors</span> <span class="o">=</span> <span class="k">new</span> <span class="n">ArrayList</span><span class="o"><</span><span class="n">Tuple</span><span class="o">>();</span> -<span class="n">anchors</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">tuple1</span><span class="o">);</span> -<span class="n">anchors</span><span class="o">.</span><span class="na">add</span><span class="o">(</span><span class="n">tuple2</span><span class="o">);</span> -<span class="n">_collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="n">anchors</span><span class="o">,</span> <span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="mi">1</span><span class="o">,</span> <span class="mi">2</span><span class="o">,</span> <span class="mi">3</span><span class="o">));</span> -</code></pre></div> -<p>Multi-anchoring adds the output tuple into multiple tuple trees. Note that it's also possible for multi-anchoring to break the tree structure and create tuple DAGs, like so:</p> - -<p><img src="images/tuple-dag.png" alt="Tuple DAG"></p> - -<p>Storm's implementation works for DAGs as well as trees (pre-release it only worked for trees, and the name "tuple tree" stuck).</p> - -<p>Anchoring is how you specify the tuple tree -- the next and final piece to Storm's reliability API is specifying when you've finished processing an individual tuple in the tuple tree. This is done by using the <code>ack</code> and <code>fail</code> methods on the <code>OutputCollector</code>. If you look back at the <code>SplitSentence</code> example, you can see that the input tuple is acked after all the word tuples are emitted.</p> - -<p>You can use the <code>fail</code> method on the <code>OutputCollector</code> to immediately fail the spout tuple at the root of the tuple tree. For example, your application may choose to catch an exception from a database client and explicitly fail the input tuple. By failing the tuple explicitly, the spout tuple can be replayed faster than if you waited for the tuple to time-out.</p> - -<p>Every tuple you process must be acked or failed. Storm uses memory to track each tuple, so if you don't ack/fail every tuple, the task will eventually run out of memory. </p> - -<p>A lot of bolts follow a common pattern of reading an input tuple, emitting tuples based on it, and then acking the tuple at the end of the <code>execute</code> method. These bolts fall into the categories of filters and simple functions. Storm has an interface called <code>BasicBolt</code> that encapsulates this pattern for you. The <code>SplitSentence</code> example can be written as a <code>BasicBolt</code> like follows:</p> -<div class="highlight"><pre><code class="language-java" data-lang="java"><span class="kd">public</span> <span class="kd">class</span> <span class="nc">SplitSentence</span> <span class="kd">extends</span> <span class="n">BaseBasicBolt</span> <span class="o">{</span> - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">execute</span><span class="o">(</span><span class="n">Tuple</span> <span class="n">tuple</span><span class="o">,</span> <span class="n">BasicOutputCollector</span> <span class="n">collector</span><span class="o">)</span> <span class="o">{</span> - <span class="n">String</span> <span class="n">sentence</span> <span class="o">=</span> <span class="n">tuple</span><span class="o">.</span><span class="na">getString</span><span class="o">(</span><span class="mi">0</span><span class="o">);</span> - <span class="k">for</span><span class="o">(</span><span class="n">String</span> <span class="nl">word:</span> <span class="n">sentence</span><span class="o">.</span><span class="na">split</span><span class="o">(</span><span class="s">" "</span><span class="o">))</span> <span class="o">{</span> - <span class="n">collector</span><span class="o">.</span><span class="na">emit</span><span class="o">(</span><span class="k">new</span> <span class="n">Values</span><span class="o">(</span><span class="n">word</span><span class="o">));</span> - <span class="o">}</span> - <span class="o">}</span> - - <span class="kd">public</span> <span class="kt">void</span> <span class="nf">declareOutputFields</span><span class="o">(</span><span class="n">OutputFieldsDeclarer</span> <span class="n">declarer</span><span class="o">)</span> <span class="o">{</span> - <span class="n">declarer</span><span class="o">.</span><span class="na">declare</span><span class="o">(</span><span class="k">new</span> <span class="n">Fields</span><span class="o">(</span><span class="s">"word"</span><span class="o">));</span> - <span class="o">}</span> - <span class="o">}</span> -</code></pre></div> -<p>This implementation is simpler than the implementation from before and is semantically identical. Tuples emitted to <code>BasicOutputCollector</code> are automatically anchored to the input tuple, and the input tuple is acked for you automatically when the execute method completes.</p> - -<p>In contrast, bolts that do aggregations or joins may delay acking a tuple until after it has computed a result based on a bunch of tuples. Aggregations and joins will commonly multi-anchor their output tuples as well. These things fall outside the simpler pattern of <code>IBasicBolt</code>.</p> - -<h3 id="how-do-i-make-my-applications-work-correctly-given-that-tuples-can-be-replayed">How do I make my applications work correctly given that tuples can be replayed?</h3> - -<p>As always in software design, the answer is "it depends." If you really want exactly once semantics use the <a href="Trident-tutorial.html">Trident</a> API. In some cases, like with a lot of analytics, dropping data is OK so disabling the fault tolerance by setting the number of acker bolts to 0 <a href="javadocs/backtype/storm/Config.html#TOPOLOGY_ACKERS">Config.TOPOLOGY_ACKERS</a>. But in some cases you want to be sure that everything was processed at least once and nothing was dropped. This is especially useful if all operations are idenpotent or if deduping can happen aferwards.</p> - -<h3 id="how-does-storm-implement-reliability-in-an-efficient-way">How does Storm implement reliability in an efficient way?</h3> - -<p>A Storm topology has a set of special "acker" tasks that track the DAG of tuples for every spout tuple. When an acker sees that a DAG is complete, it sends a message to the spout task that created the spout tuple to ack the message. You can set the number of acker tasks for a topology in the topology configuration using <a href="javadocs/backtype/storm/Config.html#TOPOLOGY_ACKERS">Config.TOPOLOGY_ACKERS</a>. Storm defaults TOPOLOGY_ACKERS to one task per worker.</p> - -<p>The best way to understand Storm's reliability implementation is to look at the lifecycle of tuples and tuple DAGs. When a tuple is created in a topology, whether in a spout or a bolt, it is given a random 64 bit id. These ids are used by ackers to track the tuple DAG for every spout tuple.</p> - -<p>Every tuple knows the ids of all the spout tuples for which it exists in their tuple trees. When you emit a new tuple in a bolt, the spout tuple ids from the tuple's anchors are copied into the new tuple. When a tuple is acked, it sends a message to the appropriate acker tasks with information about how the tuple tree changed. In particular it tells the acker "I am now completed within the tree for this spout tuple, and here are the new tuples in the tree that were anchored to me". </p> - -<p>For example, if tuples "D" and "E" were created based on tuple "C", here's how the tuple tree changes when "C" is acked: </p> - -<p><img src="images/ack_tree.png" alt="What happens on an ack"></p> - -<p>Since "C" is removed from the tree at the same time that "D" and "E" are added to it, the tree can never be prematurely completed.</p> - -<p>There are a few more details to how Storm tracks tuple trees. As mentioned already, you can have an arbitrary number of acker tasks in a topology. This leads to the following question: when a tuple is acked in the topology, how does it know to which acker task to send that information? </p> - -<p>Storm uses mod hashing to map a spout tuple id to an acker task. Since every tuple carries with it the spout tuple ids of all the trees they exist within, they know which acker tasks to communicate with. </p> - -<p>Another detail of Storm is how the acker tasks track which spout tasks are responsible for each spout tuple they're tracking. When a spout task emits a new tuple, it simply sends a message to the appropriate acker telling it that its task id is responsible for that spout tuple. Then when an acker sees a tree has been completed, it knows to which task id to send the completion message.</p> - -<p>Acker tasks do not track the tree of tuples explicitly. For large tuple trees with tens of thousands of nodes (or more), tracking all the tuple trees could overwhelm the memory used by the ackers. Instead, the ackers take a different strategy that only requires a fixed amount of space per spout tuple (about 20 bytes). This tracking algorithm is the key to how Storm works and is one of its major breakthroughs.</p> - -<p>An acker task stores a map from a spout tuple id to a pair of values. The first value is the task id that created the spout tuple which is used later on to send completion messages. The second value is a 64 bit number called the "ack val". The ack val is a representation of the state of the entire tuple tree, no matter how big or how small. It is simply the xor of all tuple ids that have been created and/or acked in the tree.</p> - -<p>When an acker task sees that an "ack val" has become 0, then it knows that the tuple tree is completed. Since tuple ids are random 64 bit numbers, the chances of an "ack val" accidentally becoming 0 is extremely small. If you work the math, at 10K acks per second, it will take 50,000,000 years until a mistake is made. And even then, it will only cause data loss if that tuple happens to fail in the topology.</p> - -<p>Now that you understand the reliability algorithm, let's go over all the failure cases and see how in each case Storm avoids data loss:</p> - -<ul> -<li><strong>A tuple isn't acked because the task died</strong>: In this case the spout tuple ids at the root of the trees for the failed tuple will time out and be replayed.</li> -<li><strong>Acker task dies</strong>: In this case all the spout tuples the acker was tracking will time out and be replayed.</li> -<li><strong>Spout task dies</strong>: In this case the source that the spout talks to is responsible for replaying the messages. For example, queues like Kestrel and RabbitMQ will place all pending messages back on the queue when a client disconnects.</li> -</ul> - -<p>As you have seen, Storm's reliability mechanisms are completely distributed, scalable, and fault-tolerant. </p> - -<h3 id="tuning-reliability">Tuning reliability</h3> - -<p>Acker tasks are lightweight, so you don't need very many of them in a topology. You can track their performance through the Storm UI (component id "__acker"). If the throughput doesn't look right, you'll need to add more acker tasks. </p> - -<p>If reliability isn't important to you -- that is, you don't care about losing tuples in failure situations -- then you can improve performance by not tracking the tuple tree for spout tuples. Not tracking a tuple tree halves the number of messages transferred since normally there's an ack message for every tuple in the tuple tree. Additionally, it requires fewer ids to be kept in each downstream tuple, reducing bandwidth usage.</p> - -<p>There are three ways to remove reliability. The first is to set Config.TOPOLOGY_ACKERS to 0. In this case, Storm will call the <code>ack</code> method on the spout immediately after the spout emits a tuple. The tuple tree won't be tracked.</p> - -<p>The second way is to remove reliability on a message by message basis. You can turn off tracking for an individual spout tuple by omitting a message id in the <code>SpoutOutputCollector.emit</code> method.</p> - -<p>Finally, if you don't care if a particular subset of the tuples downstream in the topology fail to be processed, you can emit them as unanchored tuples. Since they're not anchored to any spout tuples, they won't cause any spout tuples to fail if they aren't acked.</p> - - - - </div> - </div> - </div> -<footer> - <div class="container-fluid"> - <div class="row"> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Meetups</h5> - <ul class="latest-news"> - - <li><a href="http://www.meetup.com/Apache-Storm-Apache-Kafka/";>Apache Storm & Apache Kafka</a> <span class="small">(Sunnyvale, CA)</span></li> - - <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Apache Storm & Kafka Users</a> <span class="small">(Seattle, WA)</span></li> - - <li><a href="http://www.meetup.com/New-York-City-Storm-User-Group/";>NYC Storm User Group</a> <span class="small">(New York, NY)</span></li> - - <li><a href="http://www.meetup.com/Bay-Area-Stream-Processing";>Bay Area Stream Processing</a> <span class="small">(Emeryville, CA)</span></li> - - <li><a href="http://www.meetup.com/Boston-Storm-Users/";>Boston Realtime Data</a> <span class="small">(Boston, MA)</span></li> - - <li><a href="http://www.meetup.com/storm-london";>London Storm User Group</a> <span class="small">(London, UK)</span></li> - - <!-- <li><a href="http://www.meetup.com/Apache-Storm-Kafka-Users/";>Seatle, WA</a> <span class="small">(27 Jun 2015)</span></li> --> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>About Storm</h5> - <p>Storm integrates with any queueing system and any database system. Storm's spout abstraction makes it easy to integrate a new queuing system. Likewise, integrating Storm with database systems is easy.</p> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>First Look</h5> - <ul class="footer-list"> - <li><a href="/releases/current/Rationale.html">Rationale</a></li> - <li><a href="/releases/current/Tutorial.html">Tutorial</a></li> - <li><a href="/releases/current/Setting-up-development-environment.html">Setting up development environment</a></li> - <li><a href="/releases/current/Creating-a-new-Storm-project.html">Creating a new Storm project</a></li> - </ul> - </div> - </div> - <div class="col-md-3"> - <div class="footer-widget"> - <h5>Documentation</h5> - <ul class="footer-list"> - <li><a href="/releases/current/index.html">Index</a></li> - <li><a href="/releases/current/javadocs/index.html">Javadoc</a></li> - <li><a href="/releases/current/FAQ.html">FAQ</a></li> - </ul> - </div> - </div> - </div> - <hr/> - <div class="row"> - <div class="col-md-12"> - <p align="center">Copyright © 2015 <a href="http://www.apache.org";>Apache Software Foundation</a>. All Rights Reserved. - <br>Apache Storm, Apache, the Apache feather logo, and the Apache Storm project logos are trademarks of The Apache Software Foundation. - <br>All other marks mentioned may be trademarks or registered trademarks of their respective owners.</p> - </div> - </div> - </div> -</footer> -<!--Footer End--> -<!-- Scroll to top --> -<span class="totop"><a href="#"><i class="fa fa-angle-up"></i></a></span> - -</body> - -</html> -
