http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.html ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.html b/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.html new file mode 100644 index 0000000..7948729 --- /dev/null +++ b/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.html @@ -0,0 +1,264 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<head> +<META http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<meta content="Apache Forrest" name="Generator"> +<meta name="Forrest-version" content="0.9"> +<meta name="Forrest-skin-name" content="pelt"> +<title>Introduction to hierarchical quorums</title> +<link type="text/css" href="skin/basic.css" rel="stylesheet"> +<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet"> +<link media="print" type="text/css" href="skin/print.css" rel="stylesheet"> +<link type="text/css" href="skin/profile.css" rel="stylesheet"> +<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script> +<link rel="shortcut icon" href="images/favicon.ico"> +</head> +<body onload="init()"> +<script type="text/javascript">ndeSetTextSize();</script> +<div id="top"> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> +<a href="http://www.apache.org/";>Apache</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script> +</div> +<!--+ + |header + +--> +<div class="header"> +<!--+ + |start group logo + +--> +<div class="grouplogo"> +<a href="http://hadoop.apache.org/";><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a> +</div> +<!--+ + |end group logo + +--> +<!--+ + |start Project Logo + +--> +<div class="projectlogo"> +<a href="http://zookeeper.apache.org/";><img class="logoImage" alt="ZooKeeper" src="images/zookeeper_small.gif" title="ZooKeeper: distributed coordination"></a> +</div> +<!--+ + |end Project Logo + +--> +<!--+ + |start Search + +--> +<div class="searchbox"> +<form action="http://www.google.com/search"; method="get" class="roundtopsmall"> +<input value="zookeeper.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google"> + <input name="Search" value="Search" type="submit"> +</form> +</div> +<!--+ + |end search + +--> +<!--+ + |start Tabs + +--> +<ul id="tabs"> +<li> +<a class="unselected" href="http://zookeeper.apache.org/";>Project</a> +</li> +<li> +<a class="unselected" href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/";>Wiki</a> +</li> +<li class="current"> +<a class="selected" href="index.html">ZooKeeper 3.5 Documentation</a> +</li> +</ul> +<!--+ + |end Tabs + +--> +</div> +</div> +<div id="main"> +<div id="publishedStrip"> +<!--+ + |start Subtabs + +--> +<div id="level2tabs"></div> +<!--+ + |end Endtabs + +--> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> + + + </div> +<!--+ + |start Menu, mainarea + +--> +<!--+ + |start Menu + +--> +<div id="menu"> +<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div> +<div id="menu_1.1" class="menuitemgroup"> +<div class="menuitem"> +<a href="index.html">Welcome</a> +</div> +<div class="menuitem"> +<a href="zookeeperOver.html">Overview</a> +</div> +<div class="menuitem"> +<a href="zookeeperStarted.html">Getting Started</a> +</div> +<div class="menuitem"> +<a href="releasenotes.html">Release Notes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div> +<div id="menu_1.2" class="menuitemgroup"> +<div class="menuitem"> +<a href="api/index.html">API Docs</a> +</div> +<div class="menuitem"> +<a href="zookeeperProgrammers.html">Programmer's Guide</a> +</div> +<div class="menuitem"> +<a href="javaExample.html">Java Example</a> +</div> +<div class="menuitem"> +<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a> +</div> +<div class="menuitem"> +<a href="recipes.html">Recipes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Admin & Ops</div> +<div id="menu_1.3" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperAdmin.html">Administrator's Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperQuotas.html">Quota Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperJMX.html">JMX</a> +</div> +<div class="menuitem"> +<a href="zookeeperObservers.html">Observers Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperReconfig.html">Dynamic Reconfiguration</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Contributor</div> +<div id="menu_1.4" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperInternals.html">ZooKeeper Internals</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Miscellaneous</div> +<div id="menu_1.5" class="menuitemgroup"> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER";>Wiki</a> +</div> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/FAQ";>FAQ</a> +</div> +<div class="menuitem"> +<a href="http://zookeeper.apache.org/mailing_lists.html";>Mailing Lists</a> +</div> +</div> +<div id="credit"></div> +<div id="roundbottom"> +<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div> +<!--+ + |alternative credits + +--> +<div id="credit2"></div> +</div> +<!--+ + |end Menu + +--> +<!--+ + |start content + +--> +<div id="content"> +<div title="Portable Document Format" class="pdflink"> +<a class="dida" href="zookeeperHierarchicalQuorums.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br> + PDF</a> +</div> +<h1>Introduction to hierarchical quorums</h1> +<div id="front-matter"></div> + + + + + +<p> + This document gives an example of how to use hierarchical quorums. The basic idea is + very simple. First, we split servers into groups, and add a line for each group listing + the servers that form this group. Next we have to assign a weight to each server. + </p> + + +<p> + The following example shows how to configure a system with three groups of three servers + each, and we assign a weight of 1 to each server: + </p> + + +<pre class="code"> + group.1=1:2:3 + group.2=4:5:6 + group.3=7:8:9 + + weight.1=1 + weight.2=1 + weight.3=1 + weight.4=1 + weight.5=1 + weight.6=1 + weight.7=1 + weight.8=1 + weight.9=1 + </pre> + + +<p> + When running the system, we are able to form a quorum once we have a majority of votes from + a majority of non-zero-weight groups. Groups that have zero weight are discarded and not + considered when forming quorums. Looking at the example, we are able to form a quorum once + we have votes from at least two servers from each of two different groups. + </p> + +<p align="right"> +<font size="-2"></font> +</p> +</div> +<!--+ + |end content + +--> +<div class="clearboth"> </div> +</div> +<div id="footer"> +<!--+ + |start bottomstrip + +--> +<div class="lastmodified"> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<div class="copyright"> + Copyright © + <a href="http://www.apache.org/licenses/";>The Apache Software Foundation.</a> +</div> +<!--+ + |end bottomstrip + +--> +</div> +</body> +</html>
http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.pdf ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.pdf b/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.pdf new file mode 100644 index 0000000..c09cd2b Binary files /dev/null and b/content/doc/r3.5.4-beta/zookeeperHierarchicalQuorums.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperInternals.html ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperInternals.html b/content/doc/r3.5.4-beta/zookeeperInternals.html new file mode 100644 index 0000000..08d6ede --- /dev/null +++ b/content/doc/r3.5.4-beta/zookeeperInternals.html @@ -0,0 +1,793 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<head> +<META http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<meta content="Apache Forrest" name="Generator"> +<meta name="Forrest-version" content="0.9"> +<meta name="Forrest-skin-name" content="pelt"> +<title>ZooKeeper Internals</title> +<link type="text/css" href="skin/basic.css" rel="stylesheet"> +<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet"> +<link media="print" type="text/css" href="skin/print.css" rel="stylesheet"> +<link type="text/css" href="skin/profile.css" rel="stylesheet"> +<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script> +<link rel="shortcut icon" href="images/favicon.ico"> +</head> +<body onload="init()"> +<script type="text/javascript">ndeSetTextSize();</script> +<div id="top"> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> +<a href="http://www.apache.org/";>Apache</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script> +</div> +<!--+ + |header + +--> +<div class="header"> +<!--+ + |start group logo + +--> +<div class="grouplogo"> +<a href="http://hadoop.apache.org/";><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a> +</div> +<!--+ + |end group logo + +--> +<!--+ + |start Project Logo + +--> +<div class="projectlogo"> +<a href="http://zookeeper.apache.org/";><img class="logoImage" alt="ZooKeeper" src="images/zookeeper_small.gif" title="ZooKeeper: distributed coordination"></a> +</div> +<!--+ + |end Project Logo + +--> +<!--+ + |start Search + +--> +<div class="searchbox"> +<form action="http://www.google.com/search"; method="get" class="roundtopsmall"> +<input value="zookeeper.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google"> + <input name="Search" value="Search" type="submit"> +</form> +</div> +<!--+ + |end search + +--> +<!--+ + |start Tabs + +--> +<ul id="tabs"> +<li> +<a class="unselected" href="http://zookeeper.apache.org/";>Project</a> +</li> +<li> +<a class="unselected" href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/";>Wiki</a> +</li> +<li class="current"> +<a class="selected" href="index.html">ZooKeeper 3.5 Documentation</a> +</li> +</ul> +<!--+ + |end Tabs + +--> +</div> +</div> +<div id="main"> +<div id="publishedStrip"> +<!--+ + |start Subtabs + +--> +<div id="level2tabs"></div> +<!--+ + |end Endtabs + +--> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> + + + </div> +<!--+ + |start Menu, mainarea + +--> +<!--+ + |start Menu + +--> +<div id="menu"> +<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div> +<div id="menu_1.1" class="menuitemgroup"> +<div class="menuitem"> +<a href="index.html">Welcome</a> +</div> +<div class="menuitem"> +<a href="zookeeperOver.html">Overview</a> +</div> +<div class="menuitem"> +<a href="zookeeperStarted.html">Getting Started</a> +</div> +<div class="menuitem"> +<a href="releasenotes.html">Release Notes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div> +<div id="menu_1.2" class="menuitemgroup"> +<div class="menuitem"> +<a href="api/index.html">API Docs</a> +</div> +<div class="menuitem"> +<a href="zookeeperProgrammers.html">Programmer's Guide</a> +</div> +<div class="menuitem"> +<a href="javaExample.html">Java Example</a> +</div> +<div class="menuitem"> +<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a> +</div> +<div class="menuitem"> +<a href="recipes.html">Recipes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Admin & Ops</div> +<div id="menu_1.3" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperAdmin.html">Administrator's Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperQuotas.html">Quota Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperJMX.html">JMX</a> +</div> +<div class="menuitem"> +<a href="zookeeperObservers.html">Observers Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperReconfig.html">Dynamic Reconfiguration</a> +</div> +</div> +<div onclick="SwitchMenu('menu_selected_1.4', 'skin/')" id="menu_selected_1.4Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Contributor</div> +<div id="menu_selected_1.4" class="selectedmenuitemgroup" style="display: block;"> +<div class="menupage"> +<div class="menupagetitle">ZooKeeper Internals</div> +</div> +</div> +<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Miscellaneous</div> +<div id="menu_1.5" class="menuitemgroup"> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER";>Wiki</a> +</div> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/FAQ";>FAQ</a> +</div> +<div class="menuitem"> +<a href="http://zookeeper.apache.org/mailing_lists.html";>Mailing Lists</a> +</div> +</div> +<div id="credit"></div> +<div id="roundbottom"> +<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div> +<!--+ + |alternative credits + +--> +<div id="credit2"></div> +</div> +<!--+ + |end Menu + +--> +<!--+ + |start content + +--> +<div id="content"> +<div title="Portable Document Format" class="pdflink"> +<a class="dida" href="zookeeperInternals.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br> + PDF</a> +</div> +<h1>ZooKeeper Internals</h1> +<div id="front-matter"> +<div id="minitoc-area"> +<ul class="minitoc"> +<li> +<a href="#ch_Introduction">Introduction</a> +</li> +<li> +<a href="#sc_atomicBroadcast">Atomic Broadcast</a> +<ul class="minitoc"> +<li> +<a href="#sc_guaranteesPropertiesDefinitions">Guarantees, Properties, and Definitions</a> +</li> +<li> +<a href="#sc_leaderElection">Leader Activation</a> +</li> +<li> +<a href="#sc_activeMessaging">Active Messaging</a> +</li> +<li> +<a href="#sc_summary">Summary</a> +</li> +<li> +<a href="#sc_comparisons">Comparisons</a> +</li> +</ul> +</li> +<li> +<a href="#sc_quorum">Quorums</a> +</li> +<li> +<a href="#sc_logging">Logging</a> +<ul class="minitoc"> +<li> +<a href="#sc_developerGuidelines">Developer Guidelines</a> +<ul class="minitoc"> +<li> +<a href="#sc_rightLevel">Logging at the Right Level</a> +</li> +<li> +<a href="#sc_slf4jIdioms">Use of Standard slf4j Idioms</a> +</li> +</ul> +</li> +</ul> +</li> +</ul> +</div> +</div> + + + + + +<a name="ch_Introduction"></a> +<h2 class="h3">Introduction</h2> +<div class="section"> +<p>This document contains information on the inner workings of ZooKeeper. + So far, it discusses these topics: + </p> +<ul> + +<li> +<p> +<a href="#sc_atomicBroadcast">Atomic Broadcast</a> +</p> +</li> + +<li> +<p> +<a href="#sc_logging">Logging</a> +</p> +</li> + +</ul> +</div> + + +<a name="sc_atomicBroadcast"></a> +<h2 class="h3">Atomic Broadcast</h2> +<div class="section"> +<p> +At the heart of ZooKeeper is an atomic messaging system that keeps all of the servers in sync.</p> +<a name="sc_guaranteesPropertiesDefinitions"></a> +<h3 class="h4">Guarantees, Properties, and Definitions</h3> +<p> +The specific guarantees provided by the messaging system used by ZooKeeper are the following:</p> +<dl> + + +<dt> +<term> +<em>Reliable delivery</em> +</term> +</dt> +<dd> +<p>If a message, m, is delivered +by one server, it will be eventually delivered by all servers.</p> +</dd> + + +<dt> +<term> +<em>Total order</em> +</term> +</dt> +<dd> +<p> If a message is +delivered before message b by one server, a will be delivered before b by all +servers. If a and b are delivered messages, either a will be delivered before b +or b will be delivered before a.</p> +</dd> + + +<dt> +<term> +<em>Causal order</em> +</term> +</dt> +<dd> +<p> +If a message b is sent after a message a has been delivered by the sender of b, +a must be ordered before b. If a sender sends c after sending b, c must be ordered after b. +</p> +</dd> + + +</dl> +<p> +The ZooKeeper messaging system also needs to be efficient, reliable, and easy to +implement and maintain. We make heavy use of messaging, so we need the system to +be able to handle thousands of requests per second. Although we can require at +least k+1 correct servers to send new messages, we must be able to recover from +correlated failures such as power outages. When we implemented the system we had +little time and few engineering resources, so we needed a protocol that is +accessible to engineers and is easy to implement. We found that our protocol +satisfied all of these goals. + +</p> +<p> +Our protocol assumes that we can construct point-to-point FIFO channels between +the servers. While similar services usually assume message delivery that can +lose or reorder messages, our assumption of FIFO channels is very practical +given that we use TCP for communication. Specifically we rely on the following property of TCP:</p> +<dl> + + +<dt> +<term> +<em>Ordered delivery</em> +</term> +</dt> +<dd> +<p>Data is delivered in the same order it is sent and a message m is +delivered only after all messages sent before m have been delivered. +(The corollary to this is that if message m is lost all messages after m will be lost.)</p> +</dd> + + +<dt> +<term> +<em>No message after close</em> +</term> +</dt> +<dd> +<p>Once a FIFO channel is closed, no messages will be received from it.</p> +</dd> + + +</dl> +<p> +FLP proved that consensus cannot be achieved in asynchronous distributed systems +if failures are possible. To ensure we achieve consensus in the presence of failures +we use timeouts. However, we rely on times for liveness not for correctness. So, +if timeouts stop working (clocks malfunction for example) the messaging system may +hang, but it will not violate its guarantees.</p> +<p>When describing the ZooKeeper messaging protocol we will talk of packets, +proposals, and messages:</p> +<dl> + +<dt> +<term> +<em>Packet</em> +</term> +</dt> +<dd> +<p>a sequence of bytes sent through a FIFO channel</p> +</dd> +<dt> +<term> +<em>Proposal</em> +</term> +</dt> +<dd> +<p>a unit of agreement. Proposals are agreed upon by exchanging packets +with a quorum of ZooKeeper servers. Most proposals contain messages, however the +NEW_LEADER proposal is an example of a proposal that does not correspond to a message.</p> +</dd> +<dt> +<term> +<em>Message</em> +</term> +</dt> +<dd> +<p>a sequence of bytes to be atomically broadcast to all ZooKeeper +servers. A message put into a proposal and agreed upon before it is delivered.</p> +</dd> + + +</dl> +<p> +As stated above, ZooKeeper guarantees a total order of messages, and it also +guarantees a total order of proposals. ZooKeeper exposes the total ordering using +a ZooKeeper transaction id (<em>zxid</em>). All proposals will be stamped with a zxid when +it is proposed and exactly reflects the total ordering. Proposals are sent to all +ZooKeeper servers and committed when a quorum of them acknowledge the proposal. +If a proposal contains a message, the message will be delivered when the proposal +is committed. Acknowledgement means the server has recorded the proposal to persistent storage. +Our quorums have the requirement that any pair of quorum must have at least one server +in common. We ensure this by requiring that all quorums have size (<em>n/2+1</em>) where +n is the number of servers that make up a ZooKeeper service. +</p> +<p> +The zxid has two parts: the epoch and a counter. In our implementation the zxid +is a 64-bit number. We use the high order 32-bits for the epoch and the low order +32-bits for the counter. Because it has two parts represent the zxid both as a +number and as a pair of integers, (<em>epoch, count</em>). The epoch number represents a +change in leadership. Each time a new leader comes into power it will have its +own epoch number. We have a simple algorithm to assign a unique zxid to a proposal: +the leader simply increments the zxid to obtain a unique zxid for each proposal. +<em>Leadership activation will ensure that only one leader uses a given epoch, so our +simple algorithm guarantees that every proposal will have a unique id.</em> + +</p> +<p> +ZooKeeper messaging consists of two phases:</p> +<dl> + +<dt> +<term> +<em>Leader activation</em> +</term> +</dt> +<dd> +<p>In this phase a leader establishes the correct state of the system +and gets ready to start making proposals.</p> +</dd> + + +<dt> +<term> +<em>Active messaging</em> +</term> +</dt> +<dd> +<p>In this phase a leader accepts messages to propose and coordinates message delivery.</p> +</dd> + +</dl> +<p> +ZooKeeper is a holistic protocol. We do not focus on individual proposals, rather +look at the stream of proposals as a whole. Our strict ordering allows us to do this +efficiently and greatly simplifies our protocol. Leadership activation embodies +this holistic concept. A leader becomes active only when a quorum of followers +(The leader counts as a follower as well. You can always vote for yourself ) has synced +up with the leader, they have the same state. This state consists of all of the +proposals that the leader believes have been committed and the proposal to follow +the leader, the NEW_LEADER proposal. (Hopefully you are thinking to +yourself, <em>Does the set of proposals that the leader believes has been committed +included all the proposals that really have been committed?</em> The answer is <em>yes</em>. +Below, we make clear why.) +</p> +<a name="sc_leaderElection"></a> +<h3 class="h4">Leader Activation</h3> +<p> +Leader activation includes leader election. We currently have two leader election +algorithms in ZooKeeper: LeaderElection and FastLeaderElection (AuthFastLeaderElection +is a variant of FastLeaderElection that uses UDP and allows servers to perform a simple +form of authentication to avoid IP spoofing). ZooKeeper messaging doesn't care about the +exact method of electing a leader has long as the following holds: +</p> +<ul> + + +<li> +<p>The leader has seen the highest zxid of all the followers.</p> +</li> + +<li> +<p>A quorum of servers have committed to following the leader.</p> +</li> + + +</ul> +<p> +Of these two requirements only the first, the highest zxid amoung the followers +needs to hold for correct operation. The second requirement, a quorum of followers, +just needs to hold with high probability. We are going to recheck the second requirement, +so if a failure happens during or after the leader election and quorum is lost, +we will recover by abandoning leader activation and running another election. +</p> +<p> +After leader election a single server will be designated as a leader and start +waiting for followers to connect. The rest of the servers will try to connect to +the leader. The leader will sync up with followers by sending any proposals they +are missing, or if a follower is missing too many proposals, it will send a full +snapshot of the state to the follower. +</p> +<p> +There is a corner case in which a follower that has proposals, U, not seen +by a leader arrives. Proposals are seen in order, so the proposals of U will have a zxids +higher than zxids seen by the leader. The follower must have arrived after the +leader election, otherwise the follower would have been elected leader given that +it has seen a higher zxid. Since committed proposals must be seen by a quorum of +servers, and a quorum of servers that elected the leader did not see U, the proposals +of you have not been committed, so they can be discarded. When the follower connects +to the leader, the leader will tell the follower to discard U. +</p> +<p> +A new leader establishes a zxid to start using for new proposals by getting the +epoch, e, of the highest zxid it has seen and setting the next zxid to use to be +(e+1, 0), fter the leader syncs with a follower, it will propose a NEW_LEADER +proposal. Once the NEW_LEADER proposal has been committed, the leader will activate +and start receiving and issuing proposals. +</p> +<p> +It all sounds complicated but here are the basic rules of operation during leader +activation: +</p> +<ul> + +<li> +<p>A follower will ACK the NEW_LEADER proposal after it has synced with the leader.</p> +</li> + +<li> +<p>A follower will only ACK a NEW_LEADER proposal with a given zxid from a single server.</p> +</li> + +<li> +<p>A new leader will COMMIT the NEW_LEADER proposal when a quorum of followers have ACKed it.</p> +</li> + +<li> +<p>A follower will commit any state it received from the leader when the NEW_LEADER proposal is COMMIT.</p> +</li> + +<li> +<p>A new leader will not accept new proposals until the NEW_LEADER proposal has been COMMITED.</p> +</li> + +</ul> +<p> +If leader election terminates erroneously, we don't have a problem since the +NEW_LEADER proposal will not be committed since the leader will not have quorum. +When this happens, the leader and any remaining followers will timeout and go back +to leader election. +</p> +<a name="sc_activeMessaging"></a> +<h3 class="h4">Active Messaging</h3> +<p> +Leader Activation does all the heavy lifting. Once the leader is coronated he can +start blasting out proposals. As long as he remains the leader no other leader can +emerge since no other leader will be able to get a quorum of followers. If a new +leader does emerge, +it means that the leader has lost quorum, and the new leader will clean up any +mess left over during her leadership activation. +</p> +<p>ZooKeeper messaging operates similar to a classic two-phase commit.</p> +<img alt="" src="images/2pc.jpg"><p> +All communication channels are FIFO, so everything is done in order. Specifically +the following operating constraints are observed:</p> +<ul> + + +<li> +<p>The leader sends proposals to all followers using +the same order. Moreover, this order follows the order in which requests have been +received. Because we use FIFO channels this means that followers also receive proposals in order. +</p> +</li> + + +<li> +<p>Followers process messages in the order they are received. This +means that messages will be ACKed in order and the leader will receive ACKs from +followers in order, due to the FIFO channels. It also means that if message $m$ +has been written to non-volatile storage, all messages that were proposed before +$m$ have been written to non-volatile storage.</p> +</li> + + +<li> +<p>The leader will issue a COMMIT to all followers as soon as a +quorum of followers have ACKed a message. Since messages are ACKed in order, +COMMITs will be sent by the leader as received by the followers in order.</p> +</li> + + +<li> +<p>COMMITs are processed in order. Followers deliver a proposals +message when that proposal is committed.</p> +</li> + + +</ul> +<a name="sc_summary"></a> +<h3 class="h4">Summary</h3> +<p>So there you go. Why does it work? Specifically, why does a set of proposals +believed by a new leader always contain any proposal that has actually been committed? +First, all proposals have a unique zxid, so unlike other protocols, we never have +to worry about two different values being proposed for the same zxid; followers +(a leader is also a follower) see and record proposals in order; proposals are +committed in order; there is only one active leader at a time since followers only +follow a single leader at a time; a new leader has seen all committed proposals +from the previous epoch since it has seen the highest zxid from a quorum of servers; +any uncommited proposals from a previous epoch seen by a new leader will be committed +by that leader before it becomes active.</p> +<a name="sc_comparisons"></a> +<h3 class="h4">Comparisons</h3> +<p> +Isn't this just Multi-Paxos? No, Multi-Paxos requires some way of assuring that +there is only a single coordinator. We do not count on such assurances. Instead +we use the leader activation to recover from leadership change or old leaders +believing they are still active. +</p> +<p> +Isn't this just Paxos? Your active messaging phase looks just like phase 2 of Paxos? +Actually, to us active messaging looks just like 2 phase commit without the need to +handle aborts. Active messaging is different from both in the sense that it has +cross proposal ordering requirements. If we do not maintain strict FIFO ordering of +all packets, it all falls apart. Also, our leader activation phase is different from +both of them. In particular, our use of epochs allows us to skip blocks of uncommitted +proposals and to not worry about duplicate proposals for a given zxid. +</p> +</div> + + +<a name="sc_quorum"></a> +<h2 class="h3">Quorums</h2> +<div class="section"> +<p> +Atomic broadcast and leader election use the notion of quorum to guarantee a consistent +view of the system. By default, ZooKeeper uses majority quorums, which means that every +voting that happens in one of these protocols requires a majority to vote on. One example is +acknowledging a leader proposal: the leader can only commit once it receives an +acknowledgement from a quorum of servers. +</p> +<p> +If we extract the properties that we really need from our use of majorities, we have that we only +need to guarantee that groups of processes used to validate an operation by voting (e.g., acknowledging +a leader proposal) pairwise intersect in at least one server. Using majorities guarantees such a property. +However, there are other ways of constructing quorums different from majorities. For example, we can assign +weights to the votes of servers, and say that the votes of some servers are more important. To obtain a quorum, +we get enough votes so that the sum of weights of all votes is larger than half of the total sum of all weights. +</p> +<p> +A different construction that uses weights and is useful in wide-area deployments (co-locations) is a hierarchical +one. With this construction, we split the servers into disjoint groups and assign weights to processes. To form +a quorum, we have to get a hold of enough servers from a majority of groups G, such that for each group g in G, +the sum of votes from g is larger than half of the sum of weights in g. Interestingly, this construction enables +smaller quorums. If we have, for example, 9 servers, we split them into 3 groups, and assign a weight of 1 to each +server, then we are able to form quorums of size 4. Note that two subsets of processes composed each of a majority +of servers from each of a majority of groups necessarily have a non-empty intersection. It is reasonable to expect +that a majority of co-locations will have a majority of servers available with high probability. +</p> +<p> +With ZooKeeper, we provide a user with the ability of configuring servers to use majority quorums, weights, or a +hierarchy of groups. +</p> +</div> + + +<a name="sc_logging"></a> +<h2 class="h3">Logging</h2> +<div class="section"> +<p> +Zookeeper uses +<a href="http://www.slf4j.org/index.html";>slf4j</a> as an abstraction layer for logging. +<a href="http://logging.apache.org/log4j";>log4j</a> in version 1.2 is chosen as the final logging implementation for now. +For better embedding support, it is planned in the future to leave the decision of choosing the final logging implementation to the end user. +Therefore, always use the slf4j api to write log statements in the code, but configure log4j for how to log at runtime. +Note that slf4j has no FATAL level, former messages at FATAL level have been moved to ERROR level. +For information on configuring log4j for +ZooKeeper, see the <a href="zookeeperAdmin.html#sc_logging">Logging</a> section +of the <a href="zookeeperAdmin.html">ZooKeeper Administrator's Guide.</a> + + +</p> +<a name="sc_developerGuidelines"></a> +<h3 class="h4">Developer Guidelines</h3> +<p>Please follow the +<a href="http://www.slf4j.org/manual.html";>slf4j manual</a> when creating log statements within code. +Also read the +<a href="http://www.slf4j.org/faq.html#logging_performance";>FAQ on performance</a> +, when creating log statements. Patch reviewers will look for the following:</p> +<a name="sc_rightLevel"></a> +<h4>Logging at the Right Level</h4> +<p> +There are several levels of logging in slf4j. +It's important to pick the right one. In order of higher to lower severity:</p> +<ol> + +<li> +<p>ERROR level designates error events that might still allow the application to continue running.</p> +</li> + +<li> +<p>WARN level designates potentially harmful situations.</p> +</li> + +<li> +<p>INFO level designates informational messages that highlight the progress of the application at coarse-grained level.</p> +</li> + +<li> +<p>DEBUG Level designates fine-grained informational events that are most useful to debug an application.</p> +</li> + +<li> +<p>TRACE Level designates finer-grained informational events than the DEBUG.</p> +</li> + +</ol> +<p> +ZooKeeper is typically run in production such that log messages of INFO level +severity and higher (more severe) are output to the log.</p> +<a name="sc_slf4jIdioms"></a> +<h4>Use of Standard slf4j Idioms</h4> +<p> +<em>Static Message Logging</em> +</p> +<pre class="code"> +LOG.debug("process completed successfully!"); +</pre> +<p> +However when creating parameterized messages are required, use formatting anchors. +</p> +<pre class="code"> +LOG.debug("got {} messages in {} minutes",new Object[]{count,time}); +</pre> +<p> +<em>Naming</em> +</p> +<p> +Loggers should be named after the class in which they are used. +</p> +<pre class="code"> +public class Foo { + private static final Logger LOG = LoggerFactory.getLogger(Foo.class); + .... + public Foo() { + LOG.info("constructing Foo"); +</pre> +<p> +<em>Exception handling</em> +</p> +<pre class="code"> +try { + // code +} catch (XYZException e) { + // do this + LOG.error("Something bad happened", e); + // don't do this (generally) + // LOG.error(e); + // why? because "don't do" case hides the stack trace + + // continue process here as you need... recover or (re)throw +} +</pre> +</div> + + +<p align="right"> +<font size="-2"></font> +</p> +</div> +<!--+ + |end content + +--> +<div class="clearboth"> </div> +</div> +<div id="footer"> +<!--+ + |start bottomstrip + +--> +<div class="lastmodified"> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<div class="copyright"> + Copyright © + <a href="http://www.apache.org/licenses/";>The Apache Software Foundation.</a> +</div> +<!--+ + |end bottomstrip + +--> +</div> +</body> +</html> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperInternals.pdf ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperInternals.pdf b/content/doc/r3.5.4-beta/zookeeperInternals.pdf new file mode 100644 index 0000000..5405ff0 Binary files /dev/null and b/content/doc/r3.5.4-beta/zookeeperInternals.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperJMX.html ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperJMX.html b/content/doc/r3.5.4-beta/zookeeperJMX.html new file mode 100644 index 0000000..40f9574 --- /dev/null +++ b/content/doc/r3.5.4-beta/zookeeperJMX.html @@ -0,0 +1,467 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<head> +<META http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<meta content="Apache Forrest" name="Generator"> +<meta name="Forrest-version" content="0.9"> +<meta name="Forrest-skin-name" content="pelt"> +<title>ZooKeeper JMX</title> +<link type="text/css" href="skin/basic.css" rel="stylesheet"> +<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet"> +<link media="print" type="text/css" href="skin/print.css" rel="stylesheet"> +<link type="text/css" href="skin/profile.css" rel="stylesheet"> +<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script> +<link rel="shortcut icon" href="images/favicon.ico"> +</head> +<body onload="init()"> +<script type="text/javascript">ndeSetTextSize();</script> +<div id="top"> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> +<a href="http://www.apache.org/";>Apache</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script> +</div> +<!--+ + |header + +--> +<div class="header"> +<!--+ + |start group logo + +--> +<div class="grouplogo"> +<a href="http://hadoop.apache.org/";><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a> +</div> +<!--+ + |end group logo + +--> +<!--+ + |start Project Logo + +--> +<div class="projectlogo"> +<a href="http://zookeeper.apache.org/";><img class="logoImage" alt="ZooKeeper" src="images/zookeeper_small.gif" title="ZooKeeper: distributed coordination"></a> +</div> +<!--+ + |end Project Logo + +--> +<!--+ + |start Search + +--> +<div class="searchbox"> +<form action="http://www.google.com/search"; method="get" class="roundtopsmall"> +<input value="zookeeper.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google"> + <input name="Search" value="Search" type="submit"> +</form> +</div> +<!--+ + |end search + +--> +<!--+ + |start Tabs + +--> +<ul id="tabs"> +<li> +<a class="unselected" href="http://zookeeper.apache.org/";>Project</a> +</li> +<li> +<a class="unselected" href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/";>Wiki</a> +</li> +<li class="current"> +<a class="selected" href="index.html">ZooKeeper 3.5 Documentation</a> +</li> +</ul> +<!--+ + |end Tabs + +--> +</div> +</div> +<div id="main"> +<div id="publishedStrip"> +<!--+ + |start Subtabs + +--> +<div id="level2tabs"></div> +<!--+ + |end Endtabs + +--> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> + + + </div> +<!--+ + |start Menu, mainarea + +--> +<!--+ + |start Menu + +--> +<div id="menu"> +<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div> +<div id="menu_1.1" class="menuitemgroup"> +<div class="menuitem"> +<a href="index.html">Welcome</a> +</div> +<div class="menuitem"> +<a href="zookeeperOver.html">Overview</a> +</div> +<div class="menuitem"> +<a href="zookeeperStarted.html">Getting Started</a> +</div> +<div class="menuitem"> +<a href="releasenotes.html">Release Notes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div> +<div id="menu_1.2" class="menuitemgroup"> +<div class="menuitem"> +<a href="api/index.html">API Docs</a> +</div> +<div class="menuitem"> +<a href="zookeeperProgrammers.html">Programmer's Guide</a> +</div> +<div class="menuitem"> +<a href="javaExample.html">Java Example</a> +</div> +<div class="menuitem"> +<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a> +</div> +<div class="menuitem"> +<a href="recipes.html">Recipes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Admin & Ops</div> +<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;"> +<div class="menuitem"> +<a href="zookeeperAdmin.html">Administrator's Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperQuotas.html">Quota Guide</a> +</div> +<div class="menupage"> +<div class="menupagetitle">JMX</div> +</div> +<div class="menuitem"> +<a href="zookeeperObservers.html">Observers Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperReconfig.html">Dynamic Reconfiguration</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Contributor</div> +<div id="menu_1.4" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperInternals.html">ZooKeeper Internals</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Miscellaneous</div> +<div id="menu_1.5" class="menuitemgroup"> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER";>Wiki</a> +</div> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/FAQ";>FAQ</a> +</div> +<div class="menuitem"> +<a href="http://zookeeper.apache.org/mailing_lists.html";>Mailing Lists</a> +</div> +</div> +<div id="credit"></div> +<div id="roundbottom"> +<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div> +<!--+ + |alternative credits + +--> +<div id="credit2"></div> +</div> +<!--+ + |end Menu + +--> +<!--+ + |start content + +--> +<div id="content"> +<div title="Portable Document Format" class="pdflink"> +<a class="dida" href="zookeeperJMX.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br> + PDF</a> +</div> +<h1>ZooKeeper JMX</h1> +<div id="front-matter"> +<div id="minitoc-area"> +<ul class="minitoc"> +<li> +<a href="#ch_jmx">JMX</a> +</li> +<li> +<a href="#ch_starting">Starting ZooKeeper with JMX enabled</a> +</li> +<li> +<a href="#ch_console">Run a JMX console</a> +</li> +<li> +<a href="#ch_reference">ZooKeeper MBean Reference</a> +</li> +</ul> +</div> +</div> + + + + + +<a name="ch_jmx"></a> +<h2 class="h3">JMX</h2> +<div class="section"> +<p>Apache ZooKeeper has extensive support for JMX, allowing you + to view and manage a ZooKeeper serving ensemble.</p> +<p>This document assumes that you have basic knowledge of + JMX. See <a href="http://java.sun.com/javase/technologies/core/mntr-mgmt/javamanagement/";> + Sun JMX Technology</a> page to get started with JMX. + </p> +<p>See the <a href="http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html";> + JMX Management Guide</a> for details on setting up local and + remote management of VM instances. By default the included + <em>zkServer.sh</em> supports only local management - + review the linked document to enable support for remote management + (beyond the scope of this document). + </p> +</div> + + +<a name="ch_starting"></a> +<h2 class="h3">Starting ZooKeeper with JMX enabled</h2> +<div class="section"> +<p>The class + <em>org.apache.zookeeper.server.quorum.QuorumPeerMain</em> + will start a JMX manageable ZooKeeper server. This class + registers the proper MBeans during initalization to support JMX + monitoring and management of the + instance. See <em>bin/zkServer.sh</em> for one + example of starting ZooKeeper using QuorumPeerMain.</p> +</div> + + +<a name="ch_console"></a> +<h2 class="h3">Run a JMX console</h2> +<div class="section"> +<p>There are a number of JMX consoles available which can connect + to the running server. For this example we will use Sun's + <em>jconsole</em>.</p> +<p>The Java JDK ships with a simple JMX console + named <a href="http://java.sun.com/developer/technicalArticles/J2SE/jconsole.html";>jconsole</a> + which can be used to connect to ZooKeeper and inspect a running + server. Once you've started ZooKeeper using QuorumPeerMain + start <em>jconsole</em>, which typically resides in + <em>JDK_HOME/bin/jconsole</em> +</p> +<p>When the "new connection" window is displayed either connect + to local process (if jconsole started on same host as Server) or + use the remote process connection.</p> +<p>By default the "overview" tab for the VM is displayed (this + is a great way to get insight into the VM btw). Select + the "MBeans" tab.</p> +<p>You should now see <em>org.apache.ZooKeeperService</em> + on the left hand side. Expand this item and depending on how you've + started the server you will be able to monitor and manage various + service related features.</p> +<p>Also note that ZooKeeper will register log4j MBeans as + well. In the same section along the left hand side you will see + "log4j". Expand that to manage log4j through JMX. Of particular + interest is the ability to dynamically change the logging levels + used by editing the appender and root thresholds. Log4j MBean + registration can be disabled by passing + <em>-Dzookeeper.jmx.log4j.disable=true</em> to the JVM + when starting ZooKeeper. + </p> +</div> + + +<a name="ch_reference"></a> +<h2 class="h3">ZooKeeper MBean Reference</h2> +<div class="section"> +<p>This table details JMX for a server participating in a + replicated ZooKeeper ensemble (ie not standalone). This is the + typical case for a production environment.</p> +<table class="ForrestTable" cellspacing="1" cellpadding="4"> +<caption>MBeans, their names and description</caption> + +<title>MBeans, their names and description</title> + + + +<tr> + +<th>MBean</th> + <th>MBean Object Name</th> + <th>Description</th> + +</tr> + + +<tr> + +<td>Quorum</td> + <td>ReplicatedServer_id<#></td> + <td>Represents the Quorum, or Ensemble - parent of all + cluster members. Note that the object name includes the + "myid" of the server (name suffix) that your JMX agent has + connected to.</td> + +</tr> + +<tr> + +<td>LocalPeer|RemotePeer</td> + <td>replica.<#></td> + <td>Represents a local or remote peer (ie server + participating in the ensemble). Note that the object name + includes the "myid" of the server (name suffix).</td> + +</tr> + +<tr> + +<td>LeaderElection</td> + <td>LeaderElection</td> + <td>Represents a ZooKeeper cluster leader election which is + in progress. Provides information about the election, such as + when it started.</td> + +</tr> + +<tr> + +<td>Leader</td> + <td>Leader</td> + <td>Indicates that the parent replica is the leader and + provides attributes/operations for that server. Note that + Leader is a subclass of ZooKeeperServer, so it provides + all of the information normally associated with a + ZooKeeperServer node.</td> + +</tr> + +<tr> + +<td>Follower</td> + <td>Follower</td> + <td>Indicates that the parent replica is a follower and + provides attributes/operations for that server. Note that + Follower is a subclass of ZooKeeperServer, so it provides + all of the information normally associated with a + ZooKeeperServer node.</td> + +</tr> + +<tr> + +<td>DataTree</td> + <td>InMemoryDataTree</td> + <td>Statistics on the in memory znode database, also + operations to access finer (and more computationally + intensive) statistics on the data (such as ephemeral + count). InMemoryDataTrees are children of ZooKeeperServer + nodes.</td> + +</tr> + +<tr> + +<td>ServerCnxn</td> + <td><session_id></td> + <td>Statistics on each client connection, also + operations on those connections (such as + termination). Note the object name is the session id of + the connection in hex form.</td> + +</tr> + +</table> +<p>This table details JMX for a standalone server. Typically + standalone is only used in development situations.</p> +<table class="ForrestTable" cellspacing="1" cellpadding="4"> +<caption>MBeans, their names and description</caption> + +<title>MBeans, their names and description</title> + + + +<tr> + +<th>MBean</th> + <th>MBean Object Name</th> + <th>Description</th> + +</tr> + + +<tr> + +<td>ZooKeeperServer</td> + <td>StandaloneServer_port<#></td> + <td>Statistics on the running server, also operations + to reset these attributes. Note that the object name + includes the client port of the server (name + suffix).</td> + +</tr> + +<tr> + +<td>DataTree</td> + <td>InMemoryDataTree</td> + <td>Statistics on the in memory znode database, also + operations to access finer (and more computationally + intensive) statistics on the data (such as ephemeral + count).</td> + +</tr> + +<tr> + +<td>ServerCnxn</td> + <td><session_id></td> + <td>Statistics on each client connection, also + operations on those connections (such as + termination). Note the object name is the session id of + the connection in hex form.</td> + +</tr> + +</table> +</div> + + +<p align="right"> +<font size="-2"></font> +</p> +</div> +<!--+ + |end content + +--> +<div class="clearboth"> </div> +</div> +<div id="footer"> +<!--+ + |start bottomstrip + +--> +<div class="lastmodified"> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<div class="copyright"> + Copyright © + <a href="http://www.apache.org/licenses/";>The Apache Software Foundation.</a> +</div> +<!--+ + |end bottomstrip + +--> +</div> +</body> +</html> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperJMX.pdf ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperJMX.pdf b/content/doc/r3.5.4-beta/zookeeperJMX.pdf new file mode 100644 index 0000000..f6227ac Binary files /dev/null and b/content/doc/r3.5.4-beta/zookeeperJMX.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperObservers.html ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperObservers.html b/content/doc/r3.5.4-beta/zookeeperObservers.html new file mode 100644 index 0000000..842839c --- /dev/null +++ b/content/doc/r3.5.4-beta/zookeeperObservers.html @@ -0,0 +1,354 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<head> +<META http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<meta content="Apache Forrest" name="Generator"> +<meta name="Forrest-version" content="0.9"> +<meta name="Forrest-skin-name" content="pelt"> +<title>ZooKeeper Observers</title> +<link type="text/css" href="skin/basic.css" rel="stylesheet"> +<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet"> +<link media="print" type="text/css" href="skin/print.css" rel="stylesheet"> +<link type="text/css" href="skin/profile.css" rel="stylesheet"> +<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script> +<link rel="shortcut icon" href="images/favicon.ico"> +</head> +<body onload="init()"> +<script type="text/javascript">ndeSetTextSize();</script> +<div id="top"> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> +<a href="http://www.apache.org/";>Apache</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a> > <a href="http://zookeeper.apache.org/";>ZooKeeper</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script> +</div> +<!--+ + |header + +--> +<div class="header"> +<!--+ + |start group logo + +--> +<div class="grouplogo"> +<a href="http://hadoop.apache.org/";><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a> +</div> +<!--+ + |end group logo + +--> +<!--+ + |start Project Logo + +--> +<div class="projectlogo"> +<a href="http://zookeeper.apache.org/";><img class="logoImage" alt="ZooKeeper" src="images/zookeeper_small.gif" title="ZooKeeper: distributed coordination"></a> +</div> +<!--+ + |end Project Logo + +--> +<!--+ + |start Search + +--> +<div class="searchbox"> +<form action="http://www.google.com/search"; method="get" class="roundtopsmall"> +<input value="zookeeper.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google"> + <input name="Search" value="Search" type="submit"> +</form> +</div> +<!--+ + |end search + +--> +<!--+ + |start Tabs + +--> +<ul id="tabs"> +<li> +<a class="unselected" href="http://zookeeper.apache.org/";>Project</a> +</li> +<li> +<a class="unselected" href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/";>Wiki</a> +</li> +<li class="current"> +<a class="selected" href="index.html">ZooKeeper 3.5 Documentation</a> +</li> +</ul> +<!--+ + |end Tabs + +--> +</div> +</div> +<div id="main"> +<div id="publishedStrip"> +<!--+ + |start Subtabs + +--> +<div id="level2tabs"></div> +<!--+ + |end Endtabs + +--> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> + + + </div> +<!--+ + |start Menu, mainarea + +--> +<!--+ + |start Menu + +--> +<div id="menu"> +<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div> +<div id="menu_1.1" class="menuitemgroup"> +<div class="menuitem"> +<a href="index.html">Welcome</a> +</div> +<div class="menuitem"> +<a href="zookeeperOver.html">Overview</a> +</div> +<div class="menuitem"> +<a href="zookeeperStarted.html">Getting Started</a> +</div> +<div class="menuitem"> +<a href="releasenotes.html">Release Notes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div> +<div id="menu_1.2" class="menuitemgroup"> +<div class="menuitem"> +<a href="api/index.html">API Docs</a> +</div> +<div class="menuitem"> +<a href="zookeeperProgrammers.html">Programmer's Guide</a> +</div> +<div class="menuitem"> +<a href="javaExample.html">Java Example</a> +</div> +<div class="menuitem"> +<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a> +</div> +<div class="menuitem"> +<a href="recipes.html">Recipes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Admin & Ops</div> +<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;"> +<div class="menuitem"> +<a href="zookeeperAdmin.html">Administrator's Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperQuotas.html">Quota Guide</a> +</div> +<div class="menuitem"> +<a href="zookeeperJMX.html">JMX</a> +</div> +<div class="menupage"> +<div class="menupagetitle">Observers Guide</div> +</div> +<div class="menuitem"> +<a href="zookeeperReconfig.html">Dynamic Reconfiguration</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Contributor</div> +<div id="menu_1.4" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperInternals.html">ZooKeeper Internals</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.5', 'skin/')" id="menu_1.5Title" class="menutitle">Miscellaneous</div> +<div id="menu_1.5" class="menuitemgroup"> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER";>Wiki</a> +</div> +<div class="menuitem"> +<a href="https://cwiki.apache.org/confluence/display/ZOOKEEPER/FAQ";>FAQ</a> +</div> +<div class="menuitem"> +<a href="http://zookeeper.apache.org/mailing_lists.html";>Mailing Lists</a> +</div> +</div> +<div id="credit"></div> +<div id="roundbottom"> +<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div> +<!--+ + |alternative credits + +--> +<div id="credit2"></div> +</div> +<!--+ + |end Menu + +--> +<!--+ + |start content + +--> +<div id="content"> +<div title="Portable Document Format" class="pdflink"> +<a class="dida" href="zookeeperObservers.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br> + PDF</a> +</div> +<h1>ZooKeeper Observers</h1> +<div id="front-matter"> +<div id="minitoc-area"> +<ul class="minitoc"> +<li> +<a href="#ch_Introduction">Observers: Scaling ZooKeeper Without Hurting Write Performance + </a> +</li> +<li> +<a href="#sc_UsingObservers">How to use Observers</a> +</li> +<li> +<a href="#ch_UseCases">Example use cases</a> +</li> +</ul> +</div> +</div> + + + + + +<a name="ch_Introduction"></a> +<h2 class="h3">Observers: Scaling ZooKeeper Without Hurting Write Performance + </h2> +<div class="section"> +<p> + Although ZooKeeper performs very well by having clients connect directly + to voting members of the ensemble, this architecture makes it hard to + scale out to huge numbers of clients. The problem is that as we add more + voting members, the write performance drops. This is due to the fact that + a write operation requires the agreement of (in general) at least half the + nodes in an ensemble and therefore the cost of a vote can increase + significantly as more voters are added. + </p> +<p> + We have introduced a new type of ZooKeeper node called + an <em>Observer</em> which helps address this problem and + further improves ZooKeeper's scalability. Observers are non-voting members + of an ensemble which only hear the results of votes, not the agreement + protocol that leads up to them. Other than this simple distinction, + Observers function exactly the same as Followers - clients may connect to + them and send read and write requests to them. Observers forward these + requests to the Leader like Followers do, but they then simply wait to + hear the result of the vote. Because of this, we can increase the number + of Observers as much as we like without harming the performance of votes. + </p> +<p> + Observers have other advantages. Because they do not vote, they are not a + critical part of the ZooKeeper ensemble. Therefore they can fail, or be + disconnected from the cluster, without harming the availability of the + ZooKeeper service. The benefit to the user is that Observers may connect + over less reliable network links than Followers. In fact, Observers may be + used to talk to a ZooKeeper server from another data center. Clients of + the Observer will see fast reads, as all reads are served locally, and + writes result in minimal network traffic as the number of messages + required in the absence of the vote protocol is smaller. + </p> +</div> + +<a name="sc_UsingObservers"></a> +<h2 class="h3">How to use Observers</h2> +<div class="section"> +<p>Setting up a ZooKeeper ensemble that uses Observers is very simple, + and requires just two changes to your config files. Firstly, in the config + file of every node that is to be an Observer, you must place this line: + </p> +<pre class="code"> + peerType=observer + </pre> +<p> + This line tells ZooKeeper that the server is to be an Observer. Secondly, + in every server config file, you must add :observer to the server + definition line of each Observer. For example: + </p> +<pre class="code"> + server.1:localhost:2181:3181:observer + </pre> +<p> + This tells every other server that server.1 is an Observer, and that they + should not expect it to vote. This is all the configuration you need to do + to add an Observer to your ZooKeeper cluster. Now you can connect to it as + though it were an ordinary Follower. Try it out, by running:</p> +<pre class="code"> + $ bin/zkCli.sh -server localhost:2181 + </pre> +<p> + where localhost:2181 is the hostname and port number of the Observer as + specified in every config file. You should see a command line prompt + through which you can issue commands like <em>ls</em> to query + the ZooKeeper service. + </p> +</div> + + +<a name="ch_UseCases"></a> +<h2 class="h3">Example use cases</h2> +<div class="section"> +<p> + Two example use cases for Observers are listed below. In fact, wherever + you wish to scale the number of clients of your ZooKeeper ensemble, or + where you wish to insulate the critical part of an ensemble from the load + of dealing with client requests, Observers are a good architectural + choice. + </p> +<ul> + +<li> + +<p> As a datacenter bridge: Forming a ZK ensemble between two + datacenters is a problematic endeavour as the high variance in latency + between the datacenters could lead to false positive failure detection + and partitioning. However if the ensemble runs entirely in one + datacenter, and the second datacenter runs only Observers, partitions + aren't problematic as the ensemble remains connected. Clients of the + Observers may still see and issue proposals.</p> + +</li> + +<li> + +<p>As a link to a message bus: Some companies have expressed an + interest in using ZK as a component of a persistent reliable message + bus. Observers would give a natural integration point for this work: a + plug-in mechanism could be used to attach the stream of proposals an + Observer sees to a publish-subscribe system, again without loading the + core ensemble. + </p> + +</li> + +</ul> +</div> + +<p align="right"> +<font size="-2"></font> +</p> +</div> +<!--+ + |end content + +--> +<div class="clearboth"> </div> +</div> +<div id="footer"> +<!--+ + |start bottomstrip + +--> +<div class="lastmodified"> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<div class="copyright"> + Copyright © + <a href="http://www.apache.org/licenses/";>The Apache Software Foundation.</a> +</div> +<!--+ + |end bottomstrip + +--> +</div> +</body> +</html> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperObservers.pdf ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperObservers.pdf b/content/doc/r3.5.4-beta/zookeeperObservers.pdf new file mode 100644 index 0000000..a4dfae4 Binary files /dev/null and b/content/doc/r3.5.4-beta/zookeeperObservers.pdf differ http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperOtherInfo.html ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperOtherInfo.html b/content/doc/r3.5.4-beta/zookeeperOtherInfo.html new file mode 100644 index 0000000..24b594c --- /dev/null +++ b/content/doc/r3.5.4-beta/zookeeperOtherInfo.html @@ -0,0 +1,230 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> +<html> +<head> +<META http-equiv="Content-Type" content="text/html; charset=UTF-8"> +<meta content="Apache Forrest" name="Generator"> +<meta name="Forrest-version" content="0.8"> +<meta name="Forrest-skin-name" content="pelt"> +<title>ZooKeeper</title> +<link type="text/css" href="skin/basic.css" rel="stylesheet"> +<link media="screen" type="text/css" href="skin/screen.css" rel="stylesheet"> +<link media="print" type="text/css" href="skin/print.css" rel="stylesheet"> +<link type="text/css" href="skin/profile.css" rel="stylesheet"> +<script src="skin/getBlank.js" language="javascript" type="text/javascript"></script><script src="skin/getMenu.js" language="javascript" type="text/javascript"></script><script src="skin/fontsize.js" language="javascript" type="text/javascript"></script> +<link rel="shortcut icon" href="images/favicon.ico"> +</head> +<body onload="init()"> +<script type="text/javascript">ndeSetTextSize();</script> +<div id="top"> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> +<a href="http://www.apache.org/";>Apache</a> > <a href="http://hadoop.apache.org/";>Hadoop</a> > <a href="http://hadoop.apache.org/zookeeper/";>ZooKeeper</a><script src="skin/breadcrumbs.js" language="JavaScript" type="text/javascript"></script> +</div> +<!--+ + |header + +--> +<div class="header"> +<!--+ + |start group logo + +--> +<div class="grouplogo"> +<a href="http://hadoop.apache.org/";><img class="logoImage" alt="Hadoop" src="images/hadoop-logo.jpg" title="Apache Hadoop"></a> +</div> +<!--+ + |end group logo + +--> +<!--+ + |start Project Logo + +--> +<div class="projectlogo"> +<a href="http://hadoop.apache.org/zookeeper/";><img class="logoImage" alt="ZooKeeper" src="images/zookeeper_small.gif" title="The Hadoop database"></a> +</div> +<!--+ + |end Project Logo + +--> +<!--+ + |start Search + +--> +<div class="searchbox"> +<form action="http://www.google.com/search"; method="get" class="roundtopsmall"> +<input value="hadoop.apache.org" name="sitesearch" type="hidden"><input onFocus="getBlank (this, 'Search the site with google');" size="25" name="q" id="query" type="text" value="Search the site with google"> + <input name="Search" value="Search" type="submit"> +</form> +</div> +<!--+ + |end search + +--> +<!--+ + |start Tabs + +--> +<ul id="tabs"> +<li> +<a class="unselected" href="http://hadoop.apache.org/zookeeper/";>Project</a> +</li> +<li> +<a class="unselected" href="http://wiki.apache.org/hadoop/ZooKeeper";>Wiki</a> +</li> +<li class="current"> +<a class="selected" href="index.html">ZooKeeper Documentation</a> +</li> +</ul> +<!--+ + |end Tabs + +--> +</div> +</div> +<div id="main"> +<div id="publishedStrip"> +<!--+ + |start Subtabs + +--> +<div id="level2tabs"></div> +<!--+ + |end Endtabs + +--> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<!--+ + |breadtrail + +--> +<div class="breadtrail"> + + + </div> +<!--+ + |start Menu, mainarea + +--> +<!--+ + |start Menu + +--> +<div id="menu"> +<div onclick="SwitchMenu('menu_1.1', 'skin/')" id="menu_1.1Title" class="menutitle">Overview</div> +<div id="menu_1.1" class="menuitemgroup"> +<div class="menuitem"> +<a href="index.html">Welcome</a> +</div> +<div class="menuitem"> +<a href="zookeeperOver.html">Overview</a> +</div> +<div class="menuitem"> +<a href="zookeeperStarted.html">Getting Started</a> +</div> +<div class="menuitem"> +<a href="releasenotes.html">Release Notes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.2', 'skin/')" id="menu_1.2Title" class="menutitle">Developer</div> +<div id="menu_1.2" class="menuitemgroup"> +<div class="menuitem"> +<a href="api/index.html">API Docs</a> +</div> +<div class="menuitem"> +<a href="zookeeperProgrammers.html">Programmer's Guide</a> +</div> +<div class="menuitem"> +<a href="javaExample.html">Java Example</a> +</div> +<div class="menuitem"> +<a href="zookeeperTutorial.html">Barrier and Queue Tutorial</a> +</div> +<div class="menuitem"> +<a href="recipes.html">Recipes</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Admin & Ops</div> +<div id="menu_1.3" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperAdmin.html">Administrator's Guide</a> +</div> +</div> +<div onclick="SwitchMenu('menu_1.4', 'skin/')" id="menu_1.4Title" class="menutitle">Contributor</div> +<div id="menu_1.4" class="menuitemgroup"> +<div class="menuitem"> +<a href="zookeeperInternals.html">ZooKeeper Internals</a> +</div> +</div> +<div onclick="SwitchMenu('menu_selected_1.5', 'skin/')" id="menu_selected_1.5Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Informal Documentation</div> +<div id="menu_selected_1.5" class="selectedmenuitemgroup" style="display: block;"> +<div class="menuitem"> +<a href="http://wiki.apache.org/hadoop/ZooKeeper";>Wiki</a> +</div> +<div class="menuitem"> +<a href="http://wiki.apache.org/hadoop/ZooKeeper/FAQ";>FAQ</a> +</div> +<div class="menuitem"> +<a href="http://hadoop.apache.org/zookeeper/mailing_lists.html";>Mailing Lists</a> +</div> +<div class="menupage"> +<div class="menupagetitle">Other Info</div> +</div> +</div> +<div id="credit"></div> +<div id="roundbottom"> +<img style="display: none" class="corner" height="15" width="15" alt="" src="skin/images/rc-b-l-15-1body-2menu-3menu.png"></div> +<!--+ + |alternative credits + +--> +<div id="credit2"></div> +</div> +<!--+ + |end Menu + +--> +<!--+ + |start content + +--> +<div id="content"> +<div title="Portable Document Format" class="pdflink"> +<a class="dida" href="zookeeperOtherInfo.pdf"><img alt="PDF -icon" src="skin/images/pdfdoc.gif" class="skin"><br> + PDF</a> +</div> +<h1>ZooKeeper</h1> +<div id="minitoc-area"> +<ul class="minitoc"> +<li> +<a href="#ch_placeholder">Other Info</a> +</li> +</ul> +</div> + + + + + +<a name="N10009"></a><a name="ch_placeholder"></a> +<h2 class="h3">Other Info</h2> +<div class="section"> +<p> currently empty </p> +</div> + +<p align="right"> +<font size="-2"></font> +</p> +</div> +<!--+ + |end content + +--> +<div class="clearboth"> </div> +</div> +<div id="footer"> +<!--+ + |start bottomstrip + +--> +<div class="lastmodified"> +<script type="text/javascript"><!-- +document.write("Last Published: " + document.lastModified); +// --></script> +</div> +<div class="copyright"> + Copyright © + 2008 <a href="http://www.apache.org/licenses/";>The Apache Software Foundation.</a> +</div> +<!--+ + |end bottomstrip + +--> +</div> +</body> +</html> http://git-wip-us.apache.org/repos/asf/zookeeper/blob/ec4ec140/content/doc/r3.5.4-beta/zookeeperOtherInfo.pdf ---------------------------------------------------------------------- diff --git a/content/doc/r3.5.4-beta/zookeeperOtherInfo.pdf b/content/doc/r3.5.4-beta/zookeeperOtherInfo.pdf new file mode 100644 index 0000000..bbefa13 --- /dev/null +++ b/content/doc/r3.5.4-beta/zookeeperOtherInfo.pdf @@ -0,0 +1,151 @@ +%PDF-1.3 +%���� +4 0 obj +<< /Type /Info +/Producer (FOP 0.20.5) >> +endobj +5 0 obj +<< /Length 376 /Filter [ /ASCII85Decode /FlateDecode ] + >> +stream +Gau`Mb>,r/&A7ljp=L?eeuq2r'N#/r+G<Ne%Vb2bPtG<0!:&H?4/G[aGO0rFp<;0q4#QAF-:'%*0Jt)i</b5(37))S./[5J#\BHWH%VI*ZZGllDJb`U/"!qV:^m1VO\R4"ngI].:[AM"qMVL5''BAQP`(Mqi'M+,20!co=0Cq-g[@md)E=;bE>%,)E0Ns*L3uZ2^Z[t57XsV@TC:sfgU;83KJ!2\3;b.*./e[Q3!QrKlod0V'Tl=KR0FD?9dhi?(2EQaVS:I;<%5[hA;@CocWk6:;@S=JSIJ#tG2%GS?8]DL7$d`BF.mkIJgc]$1cBrdD-;kgPH4SaZ5]@Ygdet-qmC3Z,hm,Z#]K0$f^<SMBpT[3Ifn6SQt$~> +endstream +endobj +6 0 obj +<< /Type /Page +/Parent 1 0 R +/MediaBox [ 0 0 612 792 ] +/Resources 3 0 R +/Contents 5 0 R +/Annots 7 0 R +>> +endobj +7 0 obj +[ +8 0 R +] +endobj +8 0 obj +<< /Type /Annot +/Subtype /Link +/Rect [ 102.0 529.541 160.316 517.541 ] +/C [ 0 0 0 ] +/Border [ 0 0 0 ] +/A 9 0 R +/H /I +>> +endobj +10 0 obj +<< /Length 385 /Filter [ /ASCII85Decode /FlateDecode ] + >> +stream +Gar'$0i,\@&;>?.MAt<DY0X@_H_cHA?iWTl!^gXl[Se%L2[/m3@jaPPaO<5AhtPtZ$8Ps<5X?8%'k+<m#V8(@_nmr@)9".^iWu_$Uf3Y#)<[4(G%Z5dGBN#En.*E#\LGU40Sr9.Uc6BueThY!gRW>[*5$t9nk9g]0*-@Es2tn16M9R8fm?r4LJ'tfA(+`gk'Y9:;_b]l-4GP.KqSN+^!4^1#A]fr4CsMK4!6:IPoO6gb%47LrMIO5"<!^cSUh+7e,X_C[l-]TMAhb$R.*s>AAa\3k"<_BX\h]*EX>p&XOp`_]B8,IgKiS0kudb@92(4!VNWKQ2MnK3>`\Q9X4m$jO![MDXuBq(G;UCh3ujFIfVak?-gSP6L8N`(!1BR4[K~> +endstream +endobj +11 0 obj +<< /Type /Page +/Parent 1 0 R +/MediaBox [ 0 0 612 792 ] +/Resources 3 0 R +/Contents 10 0 R +>> +endobj +13 0 obj +<< + /Title (\376\377\0\61\0\40\0\117\0\164\0\150\0\145\0\162\0\40\0\111\0\156\0\146\0\157) + /Parent 12 0 R + /A 9 0 R +>> endobj +14 0 obj +<< /Type /Font +/Subtype /Type1 +/Name /F3 +/BaseFont /Helvetica-Bold +/Encoding /WinAnsiEncoding >> +endobj +15 0 obj +<< /Type /Font +/Subtype /Type1 +/Name /F5 +/BaseFont /Times-Roman +/Encoding /WinAnsiEncoding >> +endobj +16 0 obj +<< /Type /Font +/Subtype /Type1 +/Name /F1 +/BaseFont /Helvetica +/Encoding /WinAnsiEncoding >> +endobj +17 0 obj +<< /Type /Font +/Subtype /Type1 +/Name /F2 +/BaseFont /Helvetica-Oblique +/Encoding /WinAnsiEncoding >> +endobj +18 0 obj +<< /Type /Font +/Subtype /Type1 +/Name /F7 +/BaseFont /Times-Bold +/Encoding /WinAnsiEncoding >> +endobj +1 0 obj +<< /Type /Pages +/Count 2 +/Kids [6 0 R 11 0 R ] >> +endobj +2 0 obj +<< /Type /Catalog +/Pages 1 0 R + /Outlines 12 0 R + /PageMode /UseOutlines + >> +endobj +3 0 obj +<< +/Font << /F3 14 0 R /F5 15 0 R /F1 16 0 R /F2 17 0 R /F7 18 0 R >> +/ProcSet [ /PDF /ImageC /Text ] >> +endobj +9 0 obj +<< +/S /GoTo +/D [11 0 R /XYZ 85.0 659.0 null] +>> +endobj +12 0 obj +<< + /First 13 0 R + /Last 13 0 R +>> endobj +xref +0 19 +0000000000 65535 f +0000002095 00000 n +0000002160 00000 n +0000002252 00000 n +0000000015 00000 n +0000000071 00000 n +0000000538 00000 n +0000000658 00000 n +0000000683 00000 n +0000002375 00000 n +0000000818 00000 n +0000001295 00000 n +0000002438 00000 n +0000001403 00000 n +0000001539 00000 n +0000001652 00000 n +0000001762 00000 n +0000001870 00000 n +0000001986 00000 n +trailer +<< +/Size 19 +/Root 2 0 R +/Info 4 0 R +>> +startxref +2489 +%%EOF
