Author: phunt Date: Mon Jul 5 18:20:22 2010 New Revision: 960655 URL: http://svn.apache.org/viewvc?rev=960655&view=rev Log: ZOOKEEPER-744. Add monitoring four-letter word
Modified: hadoop/zookeeper/trunk/CHANGES.txt hadoop/zookeeper/trunk/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTree.java hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTreeBean.java hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/WatchManager.java hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/ClientBase.java hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/WatcherTest.java Modified: hadoop/zookeeper/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/CHANGES.txt?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/CHANGES.txt (original) +++ hadoop/zookeeper/trunk/CHANGES.txt Mon Jul 5 18:20:22 2010 @@ -85,6 +85,8 @@ NEW FEATURES: ZOOKEEPER-773. Log visualisation (Ivan Kelly via phunt) + ZOOKEEPER-744. Add monitoring four-letter word (Savu Andrei via phunt) + Release 3.3.0 - 2010-03-24 Non-backward compatible changes: Modified: hadoop/zookeeper/trunk/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml (original) +++ hadoop/zookeeper/trunk/src/docs/src/documentation/content/xdocs/zookeeperAdmin.xml Mon Jul 5 18:20:22 2010 @@ -1155,6 +1155,45 @@ server.3=zoo3:2888:3888</programlisting> use it carefully.</para> </listitem> </varlistentry> + + + <varlistentry> + <term>mntr</term> + + <listitem> + <para><emphasis role="bold">New in 3.4.0:</emphasis> Outputs a list + of variables that could be used for monitoring the health of the cluster.</para> + + <programlisting>$ echo mntr | nc localhost 2185 + +zk_version 3.4.0 +zk_avg_latency 0 +zk_max_latency 0 +zk_min_latency 0 +zk_packets_received 70 +zk_packets_sent 69 +zk_outstanding_requests 0 +zk_server_state leader +zk_znode_count 4 +zk_watch_count 0 +zk_ephemerals_count 0 +zk_approximate_data_size 27 +zk_followers 4 - only exposed by the Leader +zk_synced_followers 4 - only exposed by the Leader +zk_pending_syncs 0 - only exposed by the Leader +zk_open_file_descriptor_count 23 - only available on Unix platforms +zk_max_file_descriptor_count 1024 - only available on Unix platforms +</programlisting> + + <para>The output is compatible with java properties format and the content + may change over time (new keys added). Your scripts should expect changes.</para> + + <para>ATTENTION: Some of the keys are platform specific and some of the keys are only exported by the Leader. </para> + + <para>The output contains multiple lines with the following format:</para> + <programlisting>key \t value</programlisting> + </listitem> + </varlistentry> </variablelist> <para>Here's an example of the <emphasis role="bold">ruok</emphasis> Modified: hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTree.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTree.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTree.java (original) +++ hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTree.java Mon Jul 5 18:20:22 2010 @@ -243,6 +243,15 @@ public class DataTree { return dataWatches.size() + childWatches.size(); } + public int getEphemeralsCount() { + Map<Long, HashSet<String>> map = this.getEphemeralsMap(); + int result = 0; + for (HashSet<String> set : map.values()) { + result += set.size(); + } + return result; + } + /** * Get the size of the nodes based on path and data length. * Modified: hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTreeBean.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTreeBean.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTreeBean.java (original) +++ hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/DataTreeBean.java Mon Jul 5 18:20:22 2010 @@ -18,11 +18,7 @@ package org.apache.zookeeper.server; -import java.util.HashSet; -import java.util.Map; - import org.apache.zookeeper.jmx.ZKMBeanInfo; -import org.apache.zookeeper.server.DataTree; /** * This class implements the data tree MBean. @@ -43,12 +39,7 @@ public class DataTreeBean implements Dat } public int countEphemerals() { - Map<Long, HashSet<String>> map = dataTree.getEphemeralsMap(); - int result = 0; - for (HashSet<String> set : map.values()) { - result += set.size(); - } - return result; + return dataTree.getEphemeralsCount(); } public int getWatchCount() { Modified: hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java (original) +++ hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/NIOServerCnxn.java Mon Jul 5 18:20:22 2010 @@ -25,6 +25,8 @@ import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; import java.io.Writer; +import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.net.InetAddress; import java.net.InetSocketAddress; import java.nio.ByteBuffer; @@ -64,6 +66,10 @@ import org.apache.zookeeper.proto.Reques import org.apache.zookeeper.proto.WatcherEvent; import org.apache.zookeeper.server.auth.AuthenticationProvider; import org.apache.zookeeper.server.auth.ProviderRegistry; +import org.apache.zookeeper.server.quorum.Leader; +import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer; + +import com.sun.management.UnixOperatingSystemMXBean; /** * This class handles communication with clients using NIO. There is one per @@ -891,6 +897,14 @@ public class NIOServerCnxn implements Wa private final static int wchsCmd = ByteBuffer.wrap("wchs".getBytes()).getInt(); + /* + * See <a href="{...@docroot}/../../../docs/zookeeperAdmin.html#sc_zkCommands"> + * Zk Admin</a>. this link is for all the commands. + */ + private final static int mntrCmd = ByteBuffer.wrap("mntr".getBytes()) + .getInt(); + + private final static HashMap<Integer, String> cmd2String = new HashMap<Integer, String>(); @@ -910,6 +924,7 @@ public class NIOServerCnxn implements Wa cmd2String.put(wchcCmd, "wchc"); cmd2String.put(wchpCmd, "wchp"); cmd2String.put(wchsCmd, "wchs"); + cmd2String.put(mntrCmd, "mntr"); } /** @@ -1221,8 +1236,71 @@ public class NIOServerCnxn implements Wa } } } - - + + private class MonitorCommand extends CommandThread { + + MonitorCommand(PrintWriter pw) { + super(pw); + } + + @Override + public void commandRun() { + if(zk == null) { + pw.println(ZK_NOT_SERVING); + return; + } + ZKDatabase zkdb = zk.getZKDatabase(); + ServerStats stats = zk.serverStats(); + + print("version", Version.getFullVersion()); + + print("avg_latency", stats.getAvgLatency()); + print("max_latency", stats.getMaxLatency()); + print("min_latency", stats.getMinLatency()); + + print("packets_received", stats.getPacketsReceived()); + print("packets_sent", stats.getPacketsSent()); + + print("outstanding_requests", stats.getOutstandingRequests()); + + print("server_state", stats.getServerState()); + print("znode_count", zkdb.getNodeCount()); + + print("watch_count", zkdb.getDataTree().getWatchCount()); + print("ephemerals_count", zkdb.getDataTree().getEphemeralsCount()); + print("approximate_data_size", zkdb.getDataTree().approximateDataSize()); + + OperatingSystemMXBean osMbean = ManagementFactory.getOperatingSystemMXBean(); + if(osMbean != null && osMbean instanceof UnixOperatingSystemMXBean) { + UnixOperatingSystemMXBean unixos = (UnixOperatingSystemMXBean)osMbean; + + print("open_file_descriptor_count", unixos.getOpenFileDescriptorCount()); + print("max_file_descriptor_count", unixos.getMaxFileDescriptorCount()); + } + + if(stats.getServerState() == "leader") { + Leader leader = ((LeaderZooKeeperServer)zk).getLeader(); + + print("followers", leader.learners.size()); + print("synced_followers", leader.forwardingFollowers.size()); + print("pending_syncs", leader.pendingSyncs.size()); + } + } + + private void print(String key, long number) { + print(key, "" + number); + } + + private void print(String key, String value) { + pw.print("zk_"); + pw.print(key); + pw.print("\t"); + pw.println(value); + } + + } + + /** Return if four letter word found and responded to, otw false **/ private boolean checkFourLetterWord(final SelectionKey k, final int len) throws IOException @@ -1308,6 +1386,10 @@ public class NIOServerCnxn implements Wa WatchCommand wcmd = new WatchCommand(pwriter, len); wcmd.start(); return true; + } else if (len == mntrCmd) { + MonitorCommand mntr = new MonitorCommand(pwriter); + mntr.start(); + return true; } return false; } Modified: hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/WatchManager.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/WatchManager.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/WatchManager.java (original) +++ hadoop/zookeeper/trunk/src/java/main/org/apache/zookeeper/server/WatchManager.java Mon Jul 5 18:20:22 2010 @@ -44,7 +44,11 @@ public class WatchManager { new HashMap<Watcher, HashSet<String>>(); public synchronized int size(){ - return watchTable.size(); + int result = 0; + for(Set<Watcher> watches : watchTable.values()) { + result += watches.size(); + } + return result; } public synchronized void addWatch(String path, Watcher watcher) { Modified: hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/ClientBase.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/ClientBase.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/ClientBase.java (original) +++ hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/ClientBase.java Mon Jul 5 18:20:22 2010 @@ -427,6 +427,10 @@ public abstract class ClientBase extends JMXEnv.ensureOnly(); } + protected ZooKeeperServer getServer() { + return serverFactory.getZooKeeperServer(); + } + protected void tearDownAll() throws Exception { synchronized (this) { for (ZooKeeper zk : allClients) { Modified: hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java (original) +++ hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsQuorumTest.java Mon Jul 5 18:20:22 2010 @@ -89,6 +89,8 @@ public class FourLetterWordsQuorumTest e verify(hp, "stat", "Outstanding"); verify(hp, "srvr", "Outstanding"); verify(hp, "cons", "queued"); + + verify(hp, "mntr", "zk_version\t"); } } Modified: hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java (original) +++ hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/FourLetterWordsTest.java Mon Jul 5 18:20:22 2010 @@ -87,6 +87,7 @@ public class FourLetterWordsTest extends verify("stat", "Outstanding"); verify("srvr", "Outstanding"); verify("cons", "queued"); + verify("mntr", "zk_server_state\tstandalone"); } private void verify(String cmd, String expected) throws IOException { Modified: hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/WatcherTest.java URL: http://svn.apache.org/viewvc/hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/WatcherTest.java?rev=960655&r1=960654&r2=960655&view=diff ============================================================================== --- hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/WatcherTest.java (original) +++ hadoop/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/WatcherTest.java Mon Jul 5 18:20:22 2010 @@ -29,6 +29,7 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.TestableZooKeeper; import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.AsyncCallback.StatCallback; import org.apache.zookeeper.AsyncCallback.VoidCallback; @@ -128,6 +129,38 @@ public class WatcherTest extends ClientB } } + @Test + public void testWatcherCount() + throws IOException, InterruptedException, KeeperException { + ZooKeeper zk1 = null, zk2 = null; + try { + MyWatcher w1 = new MyWatcher(); + zk1 = createClient(w1, hostPort); + + MyWatcher w2 = new MyWatcher(); + zk2 = createClient(w2, hostPort); + + Stat stat = new Stat(); + zk1.create("/watch-count-test", "value".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); + zk1.create("/watch-count-test-2", "value".getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL); + + zk1.getData("/watch-count-test", w1, stat); + zk1.getData("/watch-count-test-2", w1, stat); + zk2.getData("/watch-count-test", w2, stat); + + Assert.assertEquals(getServer().getZKDatabase().getDataTree().getWatchCount(), 3); + + } finally { + if(zk1 != null) { + zk1.close(); + } + if(zk2 != null) { + zk2.close(); + } + } + + } + final static int COUNT = 100; boolean hasSeenDelete = true; /**