This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 7a73a7a377 HDDS-8868. Support CommandHandler Metrics (#4924)
7a73a7a377 is described below
commit 7a73a7a37721f3f793fe69becf5fad6f22b3a43e
Author: XiChen <[email protected]>
AuthorDate: Wed Jun 21 15:40:52 2023 +0800
HDDS-8868. Support CommandHandler Metrics (#4924)
---
.../common/helpers/CommandHandlerMetrics.java | 130 +++++++++++++++++++++
.../CloseContainerCommandHandler.java | 5 +
.../ClosePipelineCommandHandler.java | 5 +
.../commandhandler/CommandDispatcher.java | 5 +
.../commandhandler/CommandHandler.java | 33 ++++++
.../CreatePipelineCommandHandler.java | 5 +
.../commandhandler/DeleteBlocksCommandHandler.java | 16 +++
.../DeleteContainerCommandHandler.java | 5 +
.../FinalizeNewLayoutVersionCommandHandler.java | 5 +
.../ReconstructECContainersCommandHandler.java | 5 +
.../RefreshVolumeUsageCommandHandler.java | 5 +
.../ReplicateContainerCommandHandler.java | 5 +
.../SetNodeOperationalStateCommandHandler.java | 5 +
13 files changed, 229 insertions(+)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/CommandHandlerMetrics.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/CommandHandlerMetrics.java
new file mode 100644
index 0000000000..a6e4d6258d
--- /dev/null
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/CommandHandlerMetrics.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.container.common.helpers;
+
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import
org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CommandHandler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static
org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics.CommandMetricsMetricsInfo.TotalRunTimeMs;
+import static
org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics.CommandMetricsMetricsInfo.QueueWaitingTaskCount;
+import static
org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics.CommandMetricsMetricsInfo.InvocationCount;
+import static
org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics.CommandMetricsMetricsInfo.ThreadPoolActivePoolSize;
+import static
org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics.CommandMetricsMetricsInfo.ThreadPoolMaxPoolSize;
+import static
org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics.CommandMetricsMetricsInfo.CommandReceivedCount;
+
+/**
+ * This class collects and exposes metrics for CommandHandlerMetrics.
+ */
[email protected]
+public final class CommandHandlerMetrics implements MetricsSource {
+ enum CommandMetricsMetricsInfo implements MetricsInfo {
+ Command("The type of the SCM command"),
+ TotalRunTimeMs("The total runtime of the command handler in milliseconds"),
+ QueueWaitingTaskCount("The number of queued tasks waiting for execution"),
+ InvocationCount("The number of times the command handler has been
invoked"),
+ ThreadPoolActivePoolSize("The number of active threads in the thread
pool"),
+ ThreadPoolMaxPoolSize("The maximum number of threads in the thread pool"),
+ CommandReceivedCount(
+ "The number of received SCM commands for each command type");
+
+ private final String desc;
+ CommandMetricsMetricsInfo(String desc) {
+ this.desc = desc;
+ }
+
+ @Override
+ public String description() {
+ return desc;
+ }
+ }
+
+ public static final String SOURCE_NAME =
+ CommandHandlerMetrics.class.getSimpleName();
+ private final Map<Type, CommandHandler> handlerMap;
+ private final Map<Type, AtomicInteger> commandCount;
+ private CommandHandlerMetrics(Map<Type, CommandHandler> handlerMap) {
+ this.handlerMap = handlerMap;
+ this.commandCount = new HashMap<>();
+ handlerMap.forEach((k, v) -> this.commandCount.put(k, new
AtomicInteger()));
+ }
+
+ /**
+ * Creates a new instance of CommandHandlerMetrics and
+ * registers it to the DefaultMetricsSystem.
+ *
+ * @param handlerMap the map of command types to their
+ * corresponding command handlers
+ * @return the registered instance of CommandHandlerMetrics
+ */
+ public static CommandHandlerMetrics create(
+ Map<Type, CommandHandler> handlerMap) {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ return ms.register(SOURCE_NAME, "CommandHandlerMetrics Metrics",
+ new CommandHandlerMetrics(handlerMap));
+ }
+
+ /**
+ * Increases the count of received commands for the specified command type.
+ *
+ * @param type the type of the command for which the count should be
increased
+ */
+ public void increaseCommandCount(Type type) {
+ commandCount.get(type).addAndGet(1);
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ for (Map.Entry<Type, CommandHandler> entry : handlerMap.entrySet()) {
+ CommandHandler commandHandler = entry.getValue();
+ MetricsRecordBuilder builder = collector.addRecord(SOURCE_NAME)
+ .setContext("CommandHandlerMetrics")
+ .tag(CommandMetricsMetricsInfo.Command,
+ commandHandler.getCommandType().name());
+
+ builder.addGauge(TotalRunTimeMs, commandHandler.getTotalRunTime());
+ builder.addGauge(QueueWaitingTaskCount, commandHandler.getQueuedCount());
+ builder.addGauge(InvocationCount, commandHandler.getInvocationCount());
+ int activePoolSize = commandHandler.getThreadPoolActivePoolSize();
+ if (activePoolSize >= 0) {
+ builder.addGauge(ThreadPoolActivePoolSize, activePoolSize);
+ }
+ int maxPoolSize = commandHandler.getThreadPoolMaxPoolSize();
+ if (maxPoolSize >= 0) {
+ builder.addGauge(ThreadPoolMaxPoolSize, maxPoolSize);
+ }
+ builder.addGauge(CommandReceivedCount,
+ commandCount.get(commandHandler.getCommandType()).get());
+ }
+ }
+
+ public void unRegister() {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ ms.unregisterSource(SOURCE_NAME);
+ }
+}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
index 2688d9f77f..a5ae044d4f 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerCommandHandler.java
@@ -187,6 +187,11 @@ public class CloseContainerCommandHandler implements
CommandHandler {
return 0;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
+
@Override
public int getQueuedCount() {
return 0;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ClosePipelineCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ClosePipelineCommandHandler.java
index 0116721ba1..f332ad4f13 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ClosePipelineCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ClosePipelineCommandHandler.java
@@ -132,6 +132,11 @@ public class ClosePipelineCommandHandler implements
CommandHandler {
return 0;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
+
@Override
public int getQueuedCount() {
return queuedCount.get();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
index d3e8883722..ebd862ca52 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.SCMCommandProto.Type;
+import org.apache.hadoop.ozone.container.common.helpers.CommandHandlerMetrics;
import
org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
@@ -43,6 +44,7 @@ public final class CommandDispatcher {
private final Map<Type, CommandHandler> handlerMap;
private final OzoneContainer container;
private final SCMConnectionManager connectionManager;
+ private final CommandHandlerMetrics commandHandlerMetrics;
/**
* Constructs a command Dispatcher.
@@ -76,6 +78,7 @@ public final class CommandDispatcher {
}
handlerMap.put(h.getCommandType(), h);
}
+ commandHandlerMetrics = CommandHandlerMetrics.create(handlerMap);
}
public CommandHandler getCloseContainerHandler() {
@@ -96,6 +99,7 @@ public final class CommandDispatcher {
Preconditions.checkNotNull(command);
CommandHandler handler = handlerMap.get(command.getType());
if (handler != null) {
+ commandHandlerMetrics.increaseCommandCount(command.getType());
handler.handle(command, container, context, connectionManager);
} else {
LOG.error("Unknown SCM Command queued. There is no handler for this " +
@@ -108,6 +112,7 @@ public final class CommandDispatcher {
for (CommandHandler c : handlerMap.values()) {
c.stop();
}
+ commandHandlerMetrics.unRegister();
}
/**
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
index a32200de25..11735c21ce 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
@@ -62,6 +62,12 @@ public interface CommandHandler {
*/
long getAverageRunTime();
+ /**
+ * Returns the total time this function takes to run.
+ * @return long
+ */
+ long getTotalRunTime();
+
/**
* Default implementation for updating command status.
*/
@@ -86,4 +92,31 @@ public interface CommandHandler {
* @return The number of queued commands inside this handler.
*/
int getQueuedCount();
+
+ /**
+ * Returns the maximum number of threads allowed in the thread pool for this
+ * handler. If the subclass does not override this method, the default
+ * implementation will return -1, indicating that the maximum pool size is
not
+ * applicable or not defined.
+ *
+ * @return The maximum number of threads allowed in the thread pool,
+ * or -1 if not applicable or not defined.
+ */
+ default int getThreadPoolMaxPoolSize() {
+ return -1;
+ }
+
+ /**
+ * Returns the number of threads currently executing tasks in the thread pool
+ * for this handler.If the subclass does not override this method,
+ * the default implementation will return -1, indicating that the number of
+ * active threads is not applicable or not defined.
+ *
+ * @return The number of threads currently executing tasks in the thread
pool,
+ * or -1 if not applicable or not defined.
+ */
+ default int getThreadPoolActivePoolSize() {
+ return -1;
+ }
+
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
index d55e410db2..4a36a1987d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CreatePipelineCommandHandler.java
@@ -173,6 +173,11 @@ public class CreatePipelineCommandHandler implements
CommandHandler {
return 0;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
+
@Override
public int getQueuedCount() {
return queuedCount.get();
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 87e45453ea..7df4a1413c 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -65,6 +65,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
@@ -136,6 +137,16 @@ public class DeleteBlocksCommandHandler implements
CommandHandler {
return deleteCommandQueues.size();
}
+ @Override
+ public int getThreadPoolMaxPoolSize() {
+ return ((ThreadPoolExecutor)executor).getMaximumPoolSize();
+ }
+
+ @Override
+ public int getThreadPoolActivePoolSize() {
+ return ((ThreadPoolExecutor)executor).getActiveCount();
+ }
+
/**
* A delete command info.
*/
@@ -523,6 +534,11 @@ public class DeleteBlocksCommandHandler implements
CommandHandler {
return 0;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
+
@Override
public void stop() {
if (executor != null) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteContainerCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteContainerCommandHandler.java
index bca6b444b0..f2e9b748db 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteContainerCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteContainerCommandHandler.java
@@ -154,6 +154,11 @@ public class DeleteContainerCommandHandler implements
CommandHandler {
0 : totalTime.get() / invocations;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime.get();
+ }
+
@Override
public void stop() {
try {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeNewLayoutVersionCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeNewLayoutVersionCommandHandler.java
index 9f7e67409e..bd7ec5710d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeNewLayoutVersionCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/FinalizeNewLayoutVersionCommandHandler.java
@@ -119,6 +119,11 @@ public class FinalizeNewLayoutVersionCommandHandler
implements CommandHandler {
return 0;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
+
@Override
public int getQueuedCount() {
return 0;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReconstructECContainersCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReconstructECContainersCommandHandler.java
index 6c83c2de99..602687d7a0 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReconstructECContainersCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReconstructECContainersCommandHandler.java
@@ -71,6 +71,11 @@ public class ReconstructECContainersCommandHandler
implements CommandHandler {
return 0;
}
+ @Override
+ public long getTotalRunTime() {
+ return 0;
+ }
+
@Override
public int getQueuedCount() {
return supervisor
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/RefreshVolumeUsageCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/RefreshVolumeUsageCommandHandler.java
index 23934754e7..3c14b2fb16 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/RefreshVolumeUsageCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/RefreshVolumeUsageCommandHandler.java
@@ -71,6 +71,11 @@ public class RefreshVolumeUsageCommandHandler implements
CommandHandler {
0 : totalTime.get() / invocations;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime.get();
+ }
+
@Override
public int getQueuedCount() {
return 0;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
index 6e837ba55c..21b26339e2 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ReplicateContainerCommandHandler.java
@@ -111,4 +111,9 @@ public class ReplicateContainerCommandHandler implements
CommandHandler {
}
return 0;
}
+
+ @Override
+ public long getTotalRunTime() {
+ return totalTime;
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
index 2b6108ca23..6f7f4414ee 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/SetNodeOperationalStateCommandHandler.java
@@ -152,6 +152,11 @@ public class SetNodeOperationalStateCommandHandler
implements CommandHandler {
0 : totalTime.get() / invocations;
}
+ @Override
+ public long getTotalRunTime() {
+ return totalTime.get();
+ }
+
@Override
public int getQueuedCount() {
return 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
