Repository: cassandra Updated Branches: refs/heads/trunk 67e9a5ffd -> 0c5faef66
Add histogram for delay to deliver hints Patch by Jeff Jirsa; Reviewed by Stefan Podkowinski for CASSANDRA-13234 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0c5faef6 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0c5faef6 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0c5faef6 Branch: refs/heads/trunk Commit: 0c5faef664aa403998f59cc77c9b39861890cfa1 Parents: 67e9a5f Author: Jeff Jirsa <j...@jeffjirsa.net> Authored: Fri Feb 17 14:20:49 2017 -0800 Committer: Jeff Jirsa <j...@jeffjirsa.net> Committed: Sun Mar 12 21:37:11 2017 -0700 ---------------------------------------------------------------------- CHANGES.txt | 1 + doc/source/operating/metrics.rst | 25 ++++++++++++++++ .../cassandra/hints/EncodedHintMessage.java | 5 ++++ src/java/org/apache/cassandra/hints/Hint.java | 6 ++++ .../apache/cassandra/hints/HintsDispatcher.java | 3 ++ .../cassandra/metrics/HintsServiceMetrics.java | 31 ++++++++++++++++++++ 6 files changed, 71 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 3acc2b4..7247f36 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -38,6 +38,7 @@ * Conditionally update index built status to avoid unnecessary flushes (CASSANDRA-12969) * cqlsh auto completion: refactor definition of compaction strategy options (CASSANDRA-12946) * Add support for arithmetic operators (CASSANDRA-11935) + * Add histogram for delay to deliver hints (CASSANDRA-13234) 3.11.0 http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/doc/source/operating/metrics.rst ---------------------------------------------------------------------- diff --git a/doc/source/operating/metrics.rst b/doc/source/operating/metrics.rst index 373d4d2..af2e36e 100644 --- a/doc/source/operating/metrics.rst +++ b/doc/source/operating/metrics.rst @@ -524,6 +524,31 @@ Hints_created-<PeerIP> Counter Number of hints on disk for this pee Hints_not_stored-<PeerIP> Counter Number of hints not stored for this peer, due to being down past the configured hint window. =========================== ============== =========== +HintsService Metrics +^^^^^^^^^^^^^^^^^^^^^ + +Metrics specific to the Hints delivery service. There are also some metrics related to hints tracked in ``Storage Metrics`` + +These metrics include the peer endpoint **in the metric name** + +Reported name format: + +**Metric Name** + ``org.apache.cassandra.metrics.HintsService.<MetricName>`` + +**JMX MBean** + ``org.apache.cassandra.metrics:type=HintsService name=<MetricName>`` + +=========================== ============== =========== +Name Type Description +=========================== ============== =========== +HintsSucceeded Meter A meter of the hints successfully delivered +HintsFailed Meter A meter of the hints that failed deliver +HintsTimedOut Meter A meter of the hints that timed out +Hints_delays Histogram Histogram of hint delivery delays (in milliseconds) +Hints_delays-<PeerIP> Histogram Histogram of hint delivery delays (in milliseconds) per peer +=========================== ============== =========== + SSTable Index Metrics ^^^^^^^^^^^^^^^^^^^^^ http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/EncodedHintMessage.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java index 4fe05ac..50d1302 100644 --- a/src/java/org/apache/cassandra/hints/EncodedHintMessage.java +++ b/src/java/org/apache/cassandra/hints/EncodedHintMessage.java @@ -58,6 +58,11 @@ final class EncodedHintMessage return new MessageOut<>(MessagingService.Verb.HINT, this, serializer); } + public long getHintCreationTime() + { + return Hint.serializer.getHintCreationTime(hint, version); + } + private static class Serializer implements IVersionedSerializer<EncodedHintMessage> { public long serializedSize(EncodedHintMessage message, int version) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/Hint.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hints/Hint.java b/src/java/org/apache/cassandra/hints/Hint.java index 4e8f139..1582a3c 100644 --- a/src/java/org/apache/cassandra/hints/Hint.java +++ b/src/java/org/apache/cassandra/hints/Hint.java @@ -18,6 +18,7 @@ package org.apache.cassandra.hints; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -146,5 +147,10 @@ public final class Hint int gcgs = (int) in.readUnsignedVInt(); return new Hint(Mutation.serializer.deserialize(in, version), creationTime, gcgs); } + + public long getHintCreationTime(ByteBuffer hintBuffer, int version) + { + return hintBuffer.getLong(0); + } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/hints/HintsDispatcher.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/hints/HintsDispatcher.java b/src/java/org/apache/cassandra/hints/HintsDispatcher.java index 3ac77a3..6fc55df 100644 --- a/src/java/org/apache/cassandra/hints/HintsDispatcher.java +++ b/src/java/org/apache/cassandra/hints/HintsDispatcher.java @@ -27,6 +27,7 @@ import java.util.function.Function; import com.google.common.util.concurrent.RateLimiter; +import org.apache.cassandra.db.monitoring.ApproximateTime; import org.apache.cassandra.exceptions.RequestFailureReason; import org.apache.cassandra.metrics.HintsServiceMetrics; import org.apache.cassandra.net.IAsyncCallbackWithFailure; @@ -175,6 +176,7 @@ final class HintsDispatcher implements AutoCloseable { Callback callback = new Callback(); HintMessage message = new HintMessage(hostId, hint); + HintsServiceMetrics.updateDelayMetrics(address , ApproximateTime.currentTimeMillis() - hint.creationTime); MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback); return callback; } @@ -187,6 +189,7 @@ final class HintsDispatcher implements AutoCloseable { Callback callback = new Callback(); EncodedHintMessage message = new EncodedHintMessage(hostId, hint, messagingVersion); + HintsServiceMetrics.updateDelayMetrics(address, ApproximateTime.currentTimeMillis() - message.getHintCreationTime()); MessagingService.instance().sendRRWithFailure(message.createMessageOut(), address, callback); return callback; } http://git-wip-us.apache.org/repos/asf/cassandra/blob/0c5faef6/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java index ad85281..d6a75f7 100644 --- a/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java +++ b/src/java/org/apache/cassandra/metrics/HintsServiceMetrics.java @@ -17,7 +17,16 @@ */ package org.apache.cassandra.metrics; +import java.net.InetAddress; + +import com.google.common.util.concurrent.MoreExecutors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.Histogram; import com.codahale.metrics.Meter; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; @@ -26,9 +35,31 @@ import static org.apache.cassandra.metrics.CassandraMetricsRegistry.Metrics; */ public final class HintsServiceMetrics { + private static final Logger logger = LoggerFactory.getLogger(HintsServiceMetrics.class); + private static final MetricNameFactory factory = new DefaultNameFactory("HintsService"); public static final Meter hintsSucceeded = Metrics.meter(factory.createMetricName("HintsSucceeded")); public static final Meter hintsFailed = Metrics.meter(factory.createMetricName("HintsFailed")); public static final Meter hintsTimedOut = Metrics.meter(factory.createMetricName("HintsTimedOut")); + + /** Histogram of all hint delivery delays */ + private static final Histogram globalDelayHistogram = Metrics.histogram(factory.createMetricName("Hint_delays"), false); + + /** Histograms per-endpoint of hint delivery delays, This is not a cache. */ + private static final LoadingCache<InetAddress, Histogram> delayByEndpoint = Caffeine.newBuilder() + .executor(MoreExecutors.directExecutor()) + .build(address -> Metrics.histogram(factory.createMetricName("Hint_delays-"+address.getHostAddress().replace(':', '.')), false)); + + public static void updateDelayMetrics(InetAddress endpoint, long delay) + { + if (delay <= 0) + { + logger.warn("Invalid negative latency in hint delivery delay: {}", delay); + return; + } + + globalDelayHistogram.update(delay); + delayByEndpoint.get(endpoint).update(delay); + } }