This is an automated email from the ASF dual-hosted git repository.
nkalmar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 6af151a ZOOKEEPER-3366: Pluggable metrics system for ZooKeeper - move
remaining metrics to MetricsProvider
6af151a is described below
commit 6af151a7e44760c33165e58d3b33a4d2760283a4
Author: Enrico Olivelli <[email protected]>
AuthorDate: Tue Jun 4 10:55:09 2019 +0200
ZOOKEEPER-3366: Pluggable metrics system for ZooKeeper - move remaining
metrics to MetricsProvider
Migrate all remaining metrics to MetricsProvider.
We are introducing now *Gauges* which are callbacks to be called when the
Provider needs to publish current values, a Gauge is a numeric value that can
go up and down.
As during the lifecycle of a ZK server process we can have several
ZooKeeperServer instances (we have several subclasses), depending on the role
of the local peer, sometimes we have to clean up unused Gauges.
The old approach in 4lw and on http admin API was to hard code metrics,
with multiple 'instanceof' conditions.
So we introduce ZooKeeperServer#registerMetrics and
ZooKeeperServer#unregisterMetrics: these overridable functions enable each
ZooKeeperServer subclass to declare specific Gauges.
We are also introducing ZooKeeperServer#collectMonitorValues in order to
push non-metrics to monitor commands (admin and 4lw), like "server state" or
"version".
Author: Enrico Olivelli <[email protected]>
Reviewers: Norbert Kalmar <[email protected]>
Closes #918 from eolivelli/fix/other-metrics
---
.../java/org/apache/zookeeper/metrics/Gauge.java | 2 +-
.../apache/zookeeper/metrics/MetricsContext.java | 15 ++-
.../metrics/impl/DefaultMetricsProvider.java | 27 +++++-
.../metrics/impl/NullMetricsProvider.java | 7 +-
.../apache/zookeeper/server/ZooKeeperServer.java | 108 ++++++++++++++++++++-
.../apache/zookeeper/server/admin/Commands.java | 81 +---------------
.../zookeeper/server/command/MonitorCommand.java | 82 ++++------------
.../server/quorum/FollowerZooKeeperServer.java | 27 ++++++
.../server/quorum/LeaderZooKeeperServer.java | 60 ++++++++++++
.../server/quorum/ObserverZooKeeperServer.java | 8 ++
.../apache/zookeeper/server/quorum/QuorumPeer.java | 15 +++
.../server/quorum/QuorumZooKeeperServer.java | 29 ++++++
.../zookeeper/metrics/BaseTestMetricsProvider.java | 6 +-
.../zookeeper/server/admin/CommandsTest.java | 9 ++
.../apache/zookeeper/test/ObserverMasterTest.java | 18 ++--
15 files changed, 324 insertions(+), 170 deletions(-)
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java
index fb8fd67..31ad71b 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/Gauge.java
@@ -31,5 +31,5 @@ public interface Gauge {
*
* @return the current value for the gauge
*/
- long get();
+ Number get();
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java
index ba6de61..c7e7ad0 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/MetricsContext.java
@@ -52,14 +52,23 @@ public interface MetricsContext {
/**
* Registers an user provided {@link Gauge} which will be called by the
* MetricsProvider in order to sample an integer value.
+ * If another Gauge was already registered the new one will
+ * take its place.
+ * Registering a null callback is not allowed.
*
* @param name unique name of the Gauge in this context
* @param gauge the implementation of the Gauge
*
- * @return true if the Gauge was successfully registered, false if the
Gauge
- * was already registered.
*/
- boolean registerGauge(String name, Gauge gauge);
+ void registerGauge(String name, Gauge gauge);
+
+ /**
+ * Unregisters the user provided {@link Gauge} bound to the given name.
+ *
+ * @param name unique name of the Gauge in this context
+ *
+ */
+ void unregisterGauge(String name);
static enum DetailLevel {
/**
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java
index c09abc6..9a460a1 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/DefaultMetricsProvider.java
@@ -17,6 +17,7 @@
*/
package org.apache.zookeeper.metrics.impl;
+import java.util.Objects;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -33,6 +34,8 @@ import org.apache.zookeeper.server.metric.AvgMinMaxCounterSet;
import org.apache.zookeeper.server.metric.AvgMinMaxPercentileCounter;
import org.apache.zookeeper.server.metric.AvgMinMaxPercentileCounterSet;
import org.apache.zookeeper.server.metric.SimpleCounter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Default implementation of {@link MetricsProvider}.<br>
@@ -60,6 +63,8 @@ public class DefaultMetricsProvider implements
MetricsProvider {
@Override
public void stop() {
+ // release all references to external objects
+ rootMetricsContext.gauges.clear();
}
@Override
@@ -74,6 +79,7 @@ public class DefaultMetricsProvider implements
MetricsProvider {
private static final class DefaultMetricsContext implements MetricsContext
{
+ private final ConcurrentMap<String, Gauge> gauges = new
ConcurrentHashMap<>();
private final ConcurrentMap<String, SimpleCounter> counters = new
ConcurrentHashMap<>();
private final ConcurrentMap<String, AvgMinMaxCounter> basicSummaries =
new ConcurrentHashMap<>();
private final ConcurrentMap<String, AvgMinMaxPercentileCounter>
summaries = new ConcurrentHashMap<>();
@@ -82,7 +88,7 @@ public class DefaultMetricsProvider implements
MetricsProvider {
@Override
public MetricsContext getContext(String name) {
- // no hierarchy
+ // no hierarchy yet
return this;
}
@@ -94,9 +100,15 @@ public class DefaultMetricsProvider implements
MetricsProvider {
}
@Override
- public boolean registerGauge(String name, Gauge gauge) {
- // Not supported
- return false;
+ public void registerGauge(String name, Gauge gauge) {
+ Objects.requireNonNull(gauge,
+ "Cannot register a null Gauge for "+name);
+ gauges.put(name, gauge);
+ }
+
+ @Override
+ public void unregisterGauge(String name) {
+ gauges.remove(name);
}
@Override
@@ -138,6 +150,12 @@ public class DefaultMetricsProvider implements
MetricsProvider {
}
void dump(BiConsumer<String, Object> sink) {
+ gauges.forEach((name, metric) -> {
+ Number value = metric.get();
+ if (value != null) {
+ sink.accept(name, value);
+ }
+ });
counters.values().forEach(metric -> {
metric.values().forEach(sink);
});
@@ -171,6 +189,7 @@ public class DefaultMetricsProvider implements
MetricsProvider {
summarySets.values().forEach(metric -> {
metric.reset();
});
+ // no need to reset gauges
}
}
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java
index 85f01ee..89db0fd 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/metrics/impl/NullMetricsProvider.java
@@ -77,8 +77,11 @@ public class NullMetricsProvider implements MetricsProvider {
}
@Override
- public boolean registerGauge(String name, Gauge gauge) {
- return true;
+ public void registerGauge(String name, Gauge gauge) {
+ }
+
+ @Override
+ public void unregisterGauge(String name) {
}
@Override
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
index 6379f1b..5117bd8 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/ZooKeeperServer.java
@@ -36,6 +36,7 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
import javax.security.sasl.SaslException;
@@ -46,6 +47,7 @@ import org.apache.zookeeper.Environment;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.KeeperException.Code;
import org.apache.zookeeper.KeeperException.SessionExpiredException;
+import org.apache.zookeeper.Version;
import org.apache.zookeeper.ZooDefs.OpCode;
import org.apache.zookeeper.ZookeeperBanner;
import org.apache.zookeeper.common.Time;
@@ -53,6 +55,7 @@ import org.apache.zookeeper.data.ACL;
import org.apache.zookeeper.data.Id;
import org.apache.zookeeper.data.StatPersisted;
import org.apache.zookeeper.jmx.MBeanRegistry;
+import org.apache.zookeeper.metrics.MetricsContext;
import org.apache.zookeeper.proto.AuthPacket;
import org.apache.zookeeper.proto.ConnectRequest;
import org.apache.zookeeper.proto.ConnectResponse;
@@ -65,12 +68,12 @@ import
org.apache.zookeeper.server.RequestProcessor.RequestProcessorException;
import org.apache.zookeeper.server.ServerCnxn.CloseRequestException;
import org.apache.zookeeper.server.SessionTracker.Session;
import org.apache.zookeeper.server.SessionTracker.SessionExpirer;
-import org.apache.zookeeper.server.auth.AuthenticationProvider;
import org.apache.zookeeper.server.auth.ProviderRegistry;
import org.apache.zookeeper.server.auth.ServerAuthenticationProvider;
import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer;
import org.apache.zookeeper.server.util.JvmPauseMonitor;
+import org.apache.zookeeper.server.util.OSMXBean;
import org.apache.zookeeper.txn.CreateSessionTxn;
import org.apache.zookeeper.txn.TxnHeader;
import org.slf4j.Logger;
@@ -230,6 +233,7 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
+ " clientPortListenBacklog " + getClientPortListenBacklog()
+ " datadir " + txnLogFactory.getDataDir()
+ " snapdir " + txnLogFactory.getSnapDir());
+
}
public String getInitialConfig() {
@@ -558,6 +562,8 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
startJvmPauseMonitor();
+ registerMetrics();
+
setState(State.RUNNING);
notifyAll();
}
@@ -663,6 +669,11 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
// new RuntimeException("Calling shutdown").printStackTrace();
setState(State.SHUTDOWN);
+
+ // unregister all metrics that are keeping a strong reference to this
object
+ // subclasses will do their specific clean up
+ unregisterMetrics();
+
// Since sessionTracker and syncThreads poll we just have to
// set running to false and they will detect it during the poll
// interval.
@@ -1257,7 +1268,7 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
// there might be a race condition that it enabled recv after
// processing request and then disabled when check throttling.
//
- // Be aware that we're actually checking the global outstanding
+ // Be aware that we're actually checking the global outstanding
// request before this request.
//
// It's fine if the IOException thrown before we decrease the count
@@ -1437,4 +1448,97 @@ public class ZooKeeperServer implements SessionExpirer,
ServerStats.Provider {
public ResponseCache getReadResponseCache() {
return isResponseCachingEnabled ? readResponseCache : null;
}
+
+ protected void registerMetrics() {
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+
+ final ZKDatabase zkdb = this.getZKDatabase();
+ final ServerStats stats = this.serverStats();
+
+ rootContext.registerGauge("avg_latency", stats::getAvgLatency);
+
+ rootContext.registerGauge("max_latency", stats::getMaxLatency);
+ rootContext.registerGauge("min_latency", stats::getMinLatency);
+
+ rootContext.registerGauge("packets_received",
stats::getPacketsReceived);
+ rootContext.registerGauge("packets_sent", stats::getPacketsSent);
+ rootContext.registerGauge("num_alive_connections",
stats::getNumAliveClientConnections);
+
+ rootContext.registerGauge("outstanding_requests",
stats::getOutstandingRequests);
+ rootContext.registerGauge("uptime", stats::getUptime);
+
+ rootContext.registerGauge("znode_count", zkdb::getNodeCount);
+
+ rootContext.registerGauge("watch_count",
zkdb.getDataTree()::getWatchCount);
+ rootContext.registerGauge("ephemerals_count",
zkdb.getDataTree()::getEphemeralsCount);
+
+ rootContext.registerGauge("approximate_data_size",
zkdb.getDataTree()::cachedApproximateDataSize);
+
+ rootContext.registerGauge("global_sessions", zkdb::getSessionCount);
+ rootContext.registerGauge("local_sessions",
+ this.getSessionTracker()::getLocalSessionCount);
+
+ OSMXBean osMbean = new OSMXBean();
+ rootContext.registerGauge("open_file_descriptor_count",
osMbean::getOpenFileDescriptorCount);
+ rootContext.registerGauge("max_file_descriptor_count",
osMbean::getMaxFileDescriptorCount);
+ rootContext.registerGauge("connection_drop_probability",
this::getConnectionDropChance);
+
+ rootContext.registerGauge("last_client_response_size",
stats.getClientResponseStats()::getLastBufferSize);
+ rootContext.registerGauge("max_client_response_size",
stats.getClientResponseStats()::getMaxBufferSize);
+ rootContext.registerGauge("min_client_response_size",
stats.getClientResponseStats()::getMinBufferSize);
+
+ }
+
+ protected void unregisterMetrics() {
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+
+ rootContext.unregisterGauge("avg_latency");
+
+ rootContext.unregisterGauge("max_latency");
+ rootContext.unregisterGauge("min_latency");
+
+ rootContext.unregisterGauge("packets_received");
+ rootContext.unregisterGauge("packets_sent");
+ rootContext.unregisterGauge("num_alive_connections");
+
+ rootContext.unregisterGauge("outstanding_requests");
+ rootContext.unregisterGauge("uptime");
+
+ rootContext.unregisterGauge("znode_count");
+
+ rootContext.unregisterGauge("watch_count");
+ rootContext.unregisterGauge("ephemerals_count");
+ rootContext.unregisterGauge("approximate_data_size");
+
+ rootContext.unregisterGauge("global_sessions");
+ rootContext.unregisterGauge("local_sessions");
+
+ rootContext.unregisterGauge("open_file_descriptor_count");
+ rootContext.unregisterGauge("max_file_descriptor_count");
+ rootContext.unregisterGauge("connection_drop_probability");
+
+ rootContext.unregisterGauge("last_client_response_size");
+ rootContext.unregisterGauge("max_client_response_size");
+ rootContext.unregisterGauge("min_client_response_size");
+
+ }
+
+ /**
+ * Hook into admin server, useful to expose additional data
+ * that do not represent metrics.
+ *
+ * @param response a sink which collects the data.
+ */
+ public void dumpMonitorValues(BiConsumer<String, Object> response) {
+ ServerStats stats = serverStats();
+ response.accept("version", Version.getFullVersion());
+ response.accept("server_state", stats.getServerState());
+ }
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
index 4cbb476..cff9c9b 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/admin/Commands.java
@@ -36,7 +36,6 @@ import org.apache.zookeeper.server.DataTree;
import org.apache.zookeeper.server.ServerCnxnFactory;
import org.apache.zookeeper.server.ServerMetrics;
import org.apache.zookeeper.server.ServerStats;
-import org.apache.zookeeper.server.ZKDatabase;
import org.apache.zookeeper.server.ZooKeeperServer;
import org.apache.zookeeper.server.ZooTrace;
import org.apache.zookeeper.server.persistence.SnapshotInfo;
@@ -44,11 +43,8 @@ import org.apache.zookeeper.server.quorum.Follower;
import org.apache.zookeeper.server.quorum.FollowerZooKeeperServer;
import org.apache.zookeeper.server.quorum.Leader;
import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer;
-import org.apache.zookeeper.server.quorum.QuorumPeer;
-import org.apache.zookeeper.server.quorum.QuorumZooKeeperServer;
import org.apache.zookeeper.server.quorum.ObserverZooKeeperServer;
import org.apache.zookeeper.server.quorum.ReadOnlyZooKeeperServer;
-import org.apache.zookeeper.server.util.OSMXBean;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -354,83 +350,10 @@ public class Commands {
@Override
public CommandResponse run(ZooKeeperServer zkServer, Map<String,
String> kwargs) {
CommandResponse response = initializeResponse();
-
- response.put("version", Version.getFullVersion());
-
- OSMXBean osMbean = new OSMXBean();
- response.put("open_file_descriptor_count",
osMbean.getOpenFileDescriptorCount());
- response.put("max_file_descriptor_count",
osMbean.getMaxFileDescriptorCount());
-
- if (zkServer != null) {
- ZKDatabase zkdb = zkServer.getZKDatabase();
- ServerStats stats = zkServer.serverStats();
- response.put("avg_latency", stats.getAvgLatency());
- response.put("max_latency", stats.getMaxLatency());
- response.put("min_latency", stats.getMinLatency());
-
- response.put("packets_received", stats.getPacketsReceived());
- response.put("packets_sent", stats.getPacketsSent());
- response.put("num_alive_connections",
stats.getNumAliveClientConnections());
-
- response.put("outstanding_requests",
stats.getOutstandingRequests());
- response.put("uptime", stats.getUptime());
-
- response.put("server_state", stats.getServerState());
- response.put("znode_count", zkdb.getNodeCount());
-
- response.put("watch_count",
zkdb.getDataTree().getWatchCount());
- response.put("ephemerals_count",
zkdb.getDataTree().getEphemeralsCount());
- response.put("approximate_data_size",
zkdb.getDataTree().cachedApproximateDataSize());
-
- response.put("global_sessions", zkdb.getSessionCount());
- response.put("local_sessions",
- zkServer.getSessionTracker().getLocalSessionCount());
-
- response.put("connection_drop_probability",
zkServer.getConnectionDropChance());
-
- response.put("last_client_response_size",
stats.getClientResponseStats().getLastBufferSize());
- response.put("max_client_response_size",
stats.getClientResponseStats().getMaxBufferSize());
- response.put("min_client_response_size",
stats.getClientResponseStats().getMinBufferSize());
-
- if (zkServer instanceof QuorumZooKeeperServer) {
- QuorumPeer peer = ((QuorumZooKeeperServer) zkServer).self;
- response.put("quorum_size", peer.getQuorumSize());
- }
-
- if (zkServer instanceof LeaderZooKeeperServer) {
- Leader leader = ((LeaderZooKeeperServer)
zkServer).getLeader();
-
- response.put("learners", leader.getLearners().size());
- response.put("synced_followers",
leader.getForwardingFollowers().size());
- response.put("synced_non_voting_followers",
leader.getNonVotingFollowers().size());
- response.put("synced_observers",
leader.getObservingLearners().size());
- response.put("pending_syncs", leader.getNumPendingSyncs());
- response.put("leader_uptime", leader.getUptime());
-
- response.put("last_proposal_size",
leader.getProposalStats().getLastBufferSize());
- response.put("max_proposal_size",
leader.getProposalStats().getMaxBufferSize());
- response.put("min_proposal_size",
leader.getProposalStats().getMinBufferSize());
- }
-
- if (zkServer instanceof FollowerZooKeeperServer) {
- Follower follower = ((FollowerZooKeeperServer)
zkServer).getFollower();
- Integer syncedObservers = follower.getSyncedObserverSize();
- if (syncedObservers != null) {
- response.put("synced_observers", syncedObservers);
- }
- }
-
- if (zkServer instanceof ObserverZooKeeperServer) {
- response.put("observer_master_id",
((ObserverZooKeeperServer) zkServer).getObserver().getLearnerMasterId());
- }
- }
-
+ zkServer.dumpMonitorValues(response::put);
ServerMetrics.getMetrics()
.getMetricsProvider()
- .dump(
- (metric, value) -> {
- response.put(metric, value);
- });
+ .dump(response::put);
return response;
}}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java
index 8cd97da..2718d3c 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/command/MonitorCommand.java
@@ -20,14 +20,8 @@ package org.apache.zookeeper.server.command;
import java.io.PrintWriter;
-import org.apache.zookeeper.Version;
import org.apache.zookeeper.server.ServerCnxn;
import org.apache.zookeeper.server.ServerMetrics;
-import org.apache.zookeeper.server.ServerStats;
-import org.apache.zookeeper.server.ZKDatabase;
-import org.apache.zookeeper.server.quorum.Leader;
-import org.apache.zookeeper.server.quorum.LeaderZooKeeperServer;
-import org.apache.zookeeper.server.util.OSMXBean;
public class MonitorCommand extends AbstractFourLetterCommand {
@@ -41,73 +35,31 @@ public class MonitorCommand extends
AbstractFourLetterCommand {
pw.println(ZK_NOT_SERVING);
return;
}
- ZKDatabase zkdb = zkServer.getZKDatabase();
- ServerStats stats = zkServer.serverStats();
- print("version", Version.getFullVersion());
-
- print("avg_latency", stats.getAvgLatency());
- print("max_latency", stats.getMaxLatency());
- print("min_latency", stats.getMinLatency());
-
- print("packets_received", stats.getPacketsReceived());
- print("packets_sent", stats.getPacketsSent());
- print("num_alive_connections", stats.getNumAliveClientConnections());
-
- print("outstanding_requests", stats.getOutstandingRequests());
-
- print("server_state", stats.getServerState());
- print("znode_count", zkdb.getNodeCount());
-
- print("watch_count", zkdb.getDataTree().getWatchCount());
- print("ephemerals_count", zkdb.getDataTree().getEphemeralsCount());
- print("approximate_data_size",
zkdb.getDataTree().cachedApproximateDataSize());
-
- OSMXBean osMbean = new OSMXBean();
- if (osMbean != null && osMbean.getUnix() == true) {
- print("open_file_descriptor_count",
osMbean.getOpenFileDescriptorCount());
- print("max_file_descriptor_count",
osMbean.getMaxFileDescriptorCount());
- }
-
- if (stats.getServerState().equals("leader")) {
- Leader leader = ((LeaderZooKeeperServer)zkServer).getLeader();
-
- print("learners", leader.getLearners().size());
- print("synced_followers", leader.getForwardingFollowers().size());
- print("synced_non_voting_followers",
leader.getNonVotingFollowers().size());
- print("pending_syncs", leader.getNumPendingSyncs());
-
- print("last_proposal_size",
leader.getProposalStats().getLastBufferSize());
- print("max_proposal_size",
leader.getProposalStats().getMaxBufferSize());
- print("min_proposal_size",
leader.getProposalStats().getMinBufferSize());
- }
+ // non metrics
+ zkServer.dumpMonitorValues(this::print);
ServerMetrics.getMetrics()
.getMetricsProvider()
- .dump(
- (metric, value) -> {
- if (value == null) {
- print(metric, null);
- } else if (value instanceof Long
- || value instanceof Integer) {
- print(metric, ((Number) value).longValue());
- } else if (value instanceof Number) {
- print(metric, ((Number) value).doubleValue());
- } else {
- print(metric, value.toString());
- }
- });
- }
-
- private void print(String key, long number) {
- print(key, "" + number);
+ .dump(this::print);
}
- private void print(String key, double number) {
- print(key, "" + number);
+ private void print(String key, Object value) {
+ if (value == null) {
+ output(key, null);
+ } else if (value instanceof Long
+ || value instanceof Integer) {
+ // format as integers
+ output(key, value + "");
+ } else if (value instanceof Number) {
+ // format as floating point
+ output(key, ((Number) value).doubleValue() + "");
+ } else {
+ output(key, value.toString());
+ }
}
- private void print(String key, String value) {
+ private void output(String key, String value) {
pw.print("zk_");
pw.print(key);
pw.print("\t");
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java
index e8fc15f..1c439f3 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/FollowerZooKeeperServer.java
@@ -37,6 +37,8 @@ import org.apache.zookeeper.server.ServerMetrics;
import org.apache.zookeeper.txn.TxnHeader;
import javax.management.JMException;
+import org.apache.zookeeper.metrics.MetricsContext;
+import org.apache.zookeeper.server.ServerMetrics;
/**
* Just like the standard ZooKeeperServer. We just replace the request
@@ -167,4 +169,29 @@ public class FollowerZooKeeperServer extends
LearnerZooKeeperServer {
}
return false;
}
+
+ @Override
+ protected void registerMetrics() {
+ super.registerMetrics();
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+
+ rootContext.registerGauge("synced_observers",
self::getSynced_observers_metric);
+
+ }
+
+ @Override
+ protected void unregisterMetrics() {
+ super.unregisterMetrics();
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+ rootContext.unregisterGauge("synced_observers");
+
+ }
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java
index 1833474..74a738f 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/LeaderZooKeeperServer.java
@@ -33,6 +33,8 @@ import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
import javax.management.JMException;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
+import org.apache.zookeeper.metrics.MetricsContext;
+import org.apache.zookeeper.server.ServerMetrics;
/**
*
@@ -95,6 +97,64 @@ public class LeaderZooKeeperServer extends
QuorumZooKeeperServer {
}
@Override
+ protected void registerMetrics() {
+ super.registerMetrics();
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+
+ rootContext.registerGauge("learners", () -> {
+ return getLeader().getLearners().size();
+ });
+ rootContext.registerGauge("synced_followers", () -> {
+ return getLeader().getForwardingFollowers().size();
+ });
+ rootContext.registerGauge("synced_non_voting_followers", () -> {
+ return getLeader().getNonVotingFollowers().size();
+ });
+
+ rootContext.registerGauge("synced_observers",
self::getSynced_observers_metric);
+
+ rootContext.registerGauge("pending_syncs", () -> {
+ return getLeader().getNumPendingSyncs();
+ });
+ rootContext.registerGauge("leader_uptime", () -> {
+ return getLeader().getUptime();
+ });
+ rootContext.registerGauge("last_proposal_size", () -> {
+ return getLeader().getProposalStats().getLastBufferSize();
+ });
+ rootContext.registerGauge("max_proposal_size", () -> {
+ return getLeader().getProposalStats().getMaxBufferSize();
+ });
+ rootContext.registerGauge("min_proposal_size", () -> {
+ return getLeader().getProposalStats().getMinBufferSize();
+ });
+ }
+
+ @Override
+ protected void unregisterMetrics() {
+ super.unregisterMetrics();
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+ rootContext.unregisterGauge("learners");
+ rootContext.unregisterGauge("synced_followers");
+ rootContext.unregisterGauge("synced_non_voting_followers");
+ rootContext.unregisterGauge("synced_observers");
+ rootContext.unregisterGauge("pending_syncs");
+ rootContext.unregisterGauge("leader_uptime");
+
+ rootContext.unregisterGauge("last_proposal_size");
+ rootContext.unregisterGauge("max_proposal_size");
+ rootContext.unregisterGauge("min_proposal_size");
+ }
+
+ @Override
public synchronized void shutdown() {
if (containerManager != null) {
containerManager.stop();
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
index 951c1fd..e66bd61 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
@@ -19,6 +19,7 @@ package org.apache.zookeeper.server.quorum;
import java.io.IOException;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.function.BiConsumer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -141,4 +142,11 @@ public class ObserverZooKeeperServer extends
LearnerZooKeeperServer {
syncProcessor.shutdown();
}
}
+
+ @Override
+ public void dumpMonitorValues(BiConsumer<String, Object> response) {
+ super.dumpMonitorValues(response);
+ response.accept("observer_master_id",
getObserver().getLearnerMasterId());
+ }
+
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
index c2e45fd..df83df6 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
@@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import javax.security.sasl.SaslException;
+import org.apache.yetus.audience.InterfaceAudience;
import org.apache.zookeeper.KeeperException.BadArgumentsException;
import org.apache.zookeeper.common.AtomicFileWritingIdiom;
@@ -2336,4 +2337,18 @@ public class QuorumPeer extends ZooKeeperThread
implements QuorumStats.Provider
Vote vote = getCurrentVote();
return vote != null && id == vote.getId();
}
+
+ @InterfaceAudience.Private
+ /**
+ * This is a metric that depends on the status of the peer.
+ */
+ public Integer getSynced_observers_metric() {
+ if (leader != null) {
+ return leader.getObservingLearners().size();
+ } else if (follower != null) {
+ return follower.getSyncedObserverSize();
+ } else {
+ return null;
+ }
+ }
}
diff --git
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
index 72637b2..0e9559a 100644
---
a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
+++
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
@@ -26,9 +26,11 @@ import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.MultiTransactionRecord;
import org.apache.zookeeper.Op;
import org.apache.zookeeper.ZooDefs.OpCode;
+import org.apache.zookeeper.metrics.MetricsContext;
import org.apache.zookeeper.proto.CreateRequest;
import org.apache.zookeeper.server.ByteBufferInputStream;
import org.apache.zookeeper.server.Request;
+import org.apache.zookeeper.server.ServerMetrics;
import org.apache.zookeeper.server.ZKDatabase;
import org.apache.zookeeper.server.ZooKeeperServer;
import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
@@ -186,4 +188,31 @@ public abstract class QuorumZooKeeperServer extends
ZooKeeperServer {
protected void setState(State state) {
this.state = state;
}
+
+ @Override
+ protected void registerMetrics() {
+ super.registerMetrics();
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+
+ rootContext.registerGauge("quorum_size", () -> {
+ return self.getQuorumSize();
+ });
+ }
+
+ @Override
+ protected void unregisterMetrics() {
+ super.unregisterMetrics();
+
+ MetricsContext rootContext = ServerMetrics
+ .getMetrics()
+ .getMetricsProvider()
+ .getRootContext();
+
+ rootContext.unregisterGauge("quorum_size");
+ }
+
}
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java
index 7c6e1ae..3e23019 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/metrics/BaseTestMetricsProvider.java
@@ -85,10 +85,8 @@ public abstract class BaseTestMetricsProvider implements
MetricsProvider {
@Override
public MetricsContext getRootContext() {
- if (!getRootContextCalled.compareAndSet(false, true)) {
- // called twice
- throw new IllegalStateException();
- }
+ getRootContextCalled.set(true);
+
return NullMetricsProvider.NullMetricsContext.INSTANCE;
}
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
index f8813d0..be803c0 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/server/admin/CommandsTest.java
@@ -202,6 +202,15 @@ public class CommandsTest extends ClientBase {
Map<String, Object> metrics = MetricsUtils.currentServerMetrics();
for (String metric : metrics.keySet()) {
+ boolean alreadyDefined = fields
+ .stream()
+ .anyMatch(f -> {
+ return f.key.equals(metric);
+ });
+ if (alreadyDefined) {
+ // known metrics are defined statically in the block above
+ continue;
+ }
if (metric.startsWith("avg_")) {
fields.add(new Field(metric, Double.class));
} else {
diff --git
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java
index 4e07d48..f720634 100644
---
a/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java
+++
b/zookeeper-server/src/test/java/org/apache/zookeeper/test/ObserverMasterTest.java
@@ -516,34 +516,32 @@ public class ObserverMasterTest extends
QuorumPeerTestBase implements Watcher{
Assert.assertTrue("observer not emitting observer_master_id",
stats.containsKey("observer_master_id"));
// check the stats for the first peer
- stats = Commands.runCommand("mntr",
q1.getQuorumPeer().getActiveServer(), emptyMap).toMap();
if (testObserverMaster) {
if (q1.getQuorumPeer().leader == null) {
- Assert.assertEquals(1, stats.get("synced_observers"));
+ Assert.assertEquals(Integer.valueOf(1),
q1.getQuorumPeer().getSynced_observers_metric());
} else {
- Assert.assertEquals(0, stats.get("synced_observers"));
+ Assert.assertEquals(Integer.valueOf(0),
q1.getQuorumPeer().getSynced_observers_metric());
}
} else {
if (q1.getQuorumPeer().leader == null) {
- Assert.assertNull(stats.get("synced_observers"));
+
Assert.assertNull(q1.getQuorumPeer().getSynced_observers_metric());
} else {
- Assert.assertEquals(1, stats.get("synced_observers"));
+ Assert.assertEquals(Integer.valueOf(1),
q1.getQuorumPeer().getSynced_observers_metric());
}
}
// check the stats for the second peer
- stats = Commands.runCommand("mntr",
q2.getQuorumPeer().getActiveServer(), emptyMap).toMap();
if (testObserverMaster) {
if (q2.getQuorumPeer().leader == null) {
- Assert.assertEquals(1, stats.get("synced_observers"));
+ Assert.assertEquals(Integer.valueOf(1),
q2.getQuorumPeer().getSynced_observers_metric());
} else {
- Assert.assertEquals(0, stats.get("synced_observers"));
+ Assert.assertEquals(Integer.valueOf(0),
q2.getQuorumPeer().getSynced_observers_metric());
}
} else {
if (q2.getQuorumPeer().leader == null) {
- Assert.assertNull(stats.get("synced_observers"));
+
Assert.assertNull(q2.getQuorumPeer().getSynced_observers_metric());
} else {
- Assert.assertEquals(1, stats.get("synced_observers"));
+ Assert.assertEquals(Integer.valueOf(1),
q2.getQuorumPeer().getSynced_observers_metric());
}
}
