This is an automated email from the ASF dual-hosted git repository.
shashikant pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b53d19a HDDS-1783 : Latency metric for applyTransaction in
ContainerStateMachine (#1363).
b53d19a is described below
commit b53d19a343e110dbcf0ec710e9d491ec6bd77a51
Author: avijayanhwx <[email protected]>
AuthorDate: Tue Sep 3 02:48:50 2019 -0700
HDDS-1783 : Latency metric for applyTransaction in ContainerStateMachine
(#1363).
---
.../common/transport/server/ratis/CSMMetrics.java | 14 ++++++++++++++
.../transport/server/ratis/ContainerStateMachine.java | 15 ++++++++++++---
.../dist/src/main/compose/ozonesecure-mr/docker-config | 2 +-
.../common/transport/server/ratis/TestCSMMetrics.java | 16 ++++++++++++++++
4 files changed, 43 insertions(+), 4 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/CSMMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/CSMMetrics.java
index 104a433..9893ae4 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/CSMMetrics.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/CSMMetrics.java
@@ -60,6 +60,9 @@ public class CSMMetrics {
private @Metric MutableCounterLong numStartTransactionVerifyFailures;
private @Metric MutableCounterLong numContainerNotOpenVerifyFailures;
+ private @Metric MutableRate applyTransaction;
+ private @Metric MutableRate writeStateMachineData;
+
public CSMMetrics() {
int numCmdTypes = ContainerProtos.Type.values().length;
this.opsLatency = new MutableRate[numCmdTypes];
@@ -186,6 +189,10 @@ public class CSMMetrics {
return numBytesCommittedCount.value();
}
+ public MutableRate getApplyTransactionLatency() {
+ return applyTransaction;
+ }
+
public void incPipelineLatency(ContainerProtos.Type type, long latencyNanos)
{
opsLatency[type.ordinal()].add(latencyNanos);
transactionLatency.add(latencyNanos);
@@ -199,6 +206,13 @@ public class CSMMetrics {
numContainerNotOpenVerifyFailures.incr();
}
+ public void recordApplyTransactionCompletion(long latencyNanos) {
+ applyTransaction.add(latencyNanos);
+ }
+
+ public void recordWriteStateMachineCompletion(long latencyNanos) {
+ writeStateMachineData.add(latencyNanos);
+ }
public void unRegister() {
MetricsSystem ms = DefaultMetricsSystem.instance();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
index 4f876bc..0780f84 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
@@ -411,7 +411,8 @@ public class ContainerStateMachine extends BaseStateMachine
{
}
private CompletableFuture<Message> handleWriteChunk(
- ContainerCommandRequestProto requestProto, long entryIndex, long term) {
+ ContainerCommandRequestProto requestProto, long entryIndex, long term,
+ long startTime) {
final WriteChunkRequestProto write = requestProto.getWriteChunk();
RaftServer server = ratisServer.getServer();
Preconditions.checkState(server instanceof RaftServerProxy);
@@ -461,6 +462,8 @@ public class ContainerStateMachine extends BaseStateMachine
{
write.getBlockID() + " logIndex " + entryIndex + " chunkName " +
write.getChunkData().getChunkName());
raftFuture.complete(r::toByteString);
+ metrics.recordWriteStateMachineCompletion(
+ Time.monotonicNowNanos() - startTime);
}
writeChunkFutureMap.remove(entryIndex);
@@ -477,6 +480,7 @@ public class ContainerStateMachine extends BaseStateMachine
{
public CompletableFuture<Message> writeStateMachineData(LogEntryProto entry)
{
try {
metrics.incNumWriteStateMachineOps();
+ long writeStateMachineStartTime = Time.monotonicNowNanos();
ContainerCommandRequestProto requestProto =
getContainerCommandRequestProto(
entry.getStateMachineLogEntry().getLogData());
@@ -493,7 +497,7 @@ public class ContainerStateMachine extends BaseStateMachine
{
switch (cmdType) {
case WriteChunk:
return handleWriteChunk(requestProto, entry.getIndex(),
- entry.getTerm());
+ entry.getTerm(), writeStateMachineStartTime);
default:
throw new IllegalStateException("Cmd Type:" + cmdType
+ " should not have state machine data");
@@ -673,6 +677,7 @@ public class ContainerStateMachine extends BaseStateMachine
{
.setTerm(trx.getLogEntry().getTerm())
.setLogIndex(index);
+ long applyTxnStartTime = Time.monotonicNowNanos();
try {
applyTransactionSemaphore.acquire();
metrics.incNumApplyTransactionsOps();
@@ -740,7 +745,11 @@ public class ContainerStateMachine extends
BaseStateMachine {
}
}
return applyTransactionFuture;
- }).whenComplete((r, t) -> applyTransactionSemaphore.release());
+ }).whenComplete((r, t) -> {
+ applyTransactionSemaphore.release();
+ metrics.recordApplyTransactionCompletion(
+ Time.monotonicNowNanos() - applyTxnStartTime);
+ });
return applyTransactionFuture;
} catch (IOException | InterruptedException e) {
metrics.incNumApplyTransactionsFails();
diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
index 6565eef..f5c5fbd 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure-mr/docker-config
@@ -47,7 +47,7 @@ HDFS-SITE.XML_dfs.datanode.address=0.0.0.0:1019
HDFS-SITE.XML_dfs.datanode.http.address=0.0.0.0:1012
CORE-SITE.XML_dfs.data.transfer.protection=authentication
CORE-SITE.XML_hadoop.security.authentication=kerberos
-COER-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*@EXAMPLE.COM)s/@.*///L
+CORE-SITE.XML_hadoop.security.auth_to_local=RULE:[2:$1@$0](.*@EXAMPLE.COM)s/@.*///L
CORE-SITE.XML_hadoop.security.key.provider.path=kms://http@kms:9600/kms
#temporary disable authorization as
org.apache.hadoop.yarn.server.api.ResourceTrackerPB is not properly annotated
to support it
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
index 4853059..8e3c957 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/TestCSMMetrics.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.container.common.transport.server.ratis;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getDoubleGauge;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
import java.io.File;
@@ -49,6 +50,8 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import static org.apache.ratis.rpc.SupportedRpcType.GRPC;
+import static org.junit.Assert.assertTrue;
+
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.util.function.CheckedBiConsumer;
@@ -118,6 +121,12 @@ public class TestCSMMetrics {
assertCounter("NumStartTransactionVerifyFailures", 0L, metric);
assertCounter("NumContainerNotOpenVerifyFailures", 0L, metric);
assertCounter("WriteChunkNumOps", 0L, metric);
+ double applyTransactionLatency = getDoubleGauge(
+ "ApplyTransactionAvgTime", metric);
+ assertTrue(applyTransactionLatency == 0.0);
+ double writeStateMachineLatency = getDoubleGauge(
+ "WriteStateMachineDataAvgTime", metric);
+ assertTrue(writeStateMachineLatency == 0.0);
// Write Chunk
BlockID blockID = ContainerTestHelper.getTestBlockID(ContainerTestHelper.
@@ -152,6 +161,13 @@ public class TestCSMMetrics {
RaftGroupId.valueOf(pipeline.getId().getId()).toString());
assertCounter("NumQueryStateMachineOps", 1L, metric);
assertCounter("NumApplyTransactionOps", 1L, metric);
+ applyTransactionLatency = getDoubleGauge(
+ "ApplyTransactionAvgTime", metric);
+ assertTrue(applyTransactionLatency > 0.0);
+ writeStateMachineLatency = getDoubleGauge(
+ "WriteStateMachineDataAvgTime", metric);
+ assertTrue(writeStateMachineLatency > 0.0);
+
} finally {
if (client != null) {
client.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
