This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 45223caac5 HDDS-7517. Register OM-S3G gRPC performance metrics (#4044)
45223caac5 is described below
commit 45223caac5ff50949f7f9ec0d21c903334ad813e
Author: Christos Bisias <[email protected]>
AuthorDate: Tue May 23 19:38:16 2023 +0300
HDDS-7517. Register OM-S3G gRPC performance metrics (#4044)
---
hadoop-hdds/common/pom.xml | 6 +
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +
.../hadoop/ozone/grpc/metrics/GrpcMetrics.java | 231 +++++++++++++++++++++
.../GrpcMetricsServerRequestInterceptor.java | 98 +++++++++
.../GrpcMetricsServerResponseInterceptor.java | 64 ++++++
.../metrics/GrpcMetricsServerTransportFilter.java | 47 +++++
.../hadoop/ozone/grpc/metrics/package-info.java | 23 ++
hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 +
.../main/smoketest/grpc/grpc-om-s3-metrics.robot | 113 ++++++++++
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 +
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 30 ++-
.../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 4 +-
12 files changed, 610 insertions(+), 11 deletions(-)
diff --git a/hadoop-hdds/common/pom.xml b/hadoop-hdds/common/pom.xml
index d63515b3b7..ac63fe3bb3 100644
--- a/hadoop-hdds/common/pom.xml
+++ b/hadoop-hdds/common/pom.xml
@@ -242,6 +242,12 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd";>
<artifactId>spotbugs-annotations</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-api</artifactId>
+ <version>${io.grpc.version}</version>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index 96f9904732..9493dcb1a7 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -164,6 +164,8 @@ public final class OzoneConfigKeys {
public static final String OZONE_CLIENT_EC_GRPC_RETRIES_MAX =
"ozone.client.ec.grpc.retries.max";
public static final int OZONE_CLIENT_EC_GRPC_RETRIES_MAX_DEFAULT = 3;
+ public static final String OZONE_GPRC_METRICS_PERCENTILES_INTERVALS_KEY
+ = "ozone.grpc.metrics.percentiles.intervals";
/**
* Ozone administrator users delimited by comma.
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
new file mode 100644
index 0000000000..3b2f7b5555
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetrics.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.grpc.metrics;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.Interns;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Class which maintains metrics related to using GRPC.
+ */
+@Metrics(about = "GRPC Metrics", context = OzoneConsts.OZONE)
+public class GrpcMetrics implements MetricsSource {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GrpcMetrics.class);
+
+ private static final MetricsInfo LATEST_REQUEST_TYPE = Interns
+ .info(
+ "LatestRequestType",
+ "Latest type of request for " +
+ "which metrics were captured");
+
+ private static final String SOURCE_NAME =
+ GrpcMetrics.class.getSimpleName();
+
+ private final MetricsRegistry registry;
+ private final boolean grpcQuantileEnable;
+ private String requestType;
+
+ public GrpcMetrics(Configuration conf) {
+ this.registry = new MetricsRegistry("grpc");
+ this.requestType = "NoRequest";
+ int[] intervals = conf.getInts(
+ OzoneConfigKeys.OZONE_GPRC_METRICS_PERCENTILES_INTERVALS_KEY);
+ grpcQuantileEnable = (intervals.length > 0);
+ if (grpcQuantileEnable) {
+ grpcQueueTimeMillisQuantiles =
+ new MutableQuantiles[intervals.length];
+ grpcProcessingTimeMillisQuantiles =
+ new MutableQuantiles[intervals.length];
+ for (int i = 0; i < intervals.length; i++) {
+ int interval = intervals[i];
+ grpcProcessingTimeMillisQuantiles[i] = registry
+ .newQuantiles("grpcQueueTime" + interval
+ + "s", "grpc queue time in millisecond", "ops",
+ "latency", interval);
+ grpcProcessingTimeMillisQuantiles[i] = registry.newQuantiles(
+ "grpcProcessingTime" + interval + "s",
+ "grpc processing time in millisecond",
+ "ops", "latency", interval);
+ }
+ }
+ LOG.debug("Initialized " + registry);
+ }
+
+ /**
+ * Create and return GrpcMetrics instance.
+ * @param conf
+ * @return GrpcMetrics
+ */
+ public static synchronized GrpcMetrics create(Configuration conf) {
+ GrpcMetrics metrics = new GrpcMetrics(conf);
+ return DefaultMetricsSystem.instance().register(SOURCE_NAME,
+ "Metrics for using gRPC", metrics);
+ }
+
+ /**
+ * Unregister the metrics instance.
+ */
+ public void unRegister() {
+ DefaultMetricsSystem.instance().unregisterSource(SOURCE_NAME);
+ }
+
+ @Override
+ public synchronized void getMetrics(MetricsCollector collector, boolean all)
{
+ MetricsRecordBuilder recordBuilder = collector.addRecord(SOURCE_NAME);
+
+ recordBuilder.tag(LATEST_REQUEST_TYPE, requestType);
+
+ sentBytes.snapshot(recordBuilder, all);
+ receivedBytes.snapshot(recordBuilder, all);
+ unknownMessagesSent.snapshot(recordBuilder, all);
+ unknownMessagesReceived.snapshot(recordBuilder, all);
+ grpcQueueTime.snapshot(recordBuilder, all);
+ grpcProcessingTime.snapshot(recordBuilder, all);
+ numOpenClientConnections.snapshot(recordBuilder, all);
+ recordBuilder.endRecord();
+ }
+
+ @Metric("Number of sent bytes")
+ private MutableCounterLong sentBytes;
+
+ @Metric("Number of received bytes")
+ private MutableCounterLong receivedBytes;
+
+ @Metric("Number of unknown messages sent")
+ private MutableCounterLong unknownMessagesSent;
+
+ @Metric("Number of unknown messages received")
+ private MutableCounterLong unknownMessagesReceived;
+
+ @Metric("Queue time")
+ private MutableRate grpcQueueTime;
+
+ // There should be no getter method to avoid
+ // exposing internal representation. FindBugs error raised.
+ private MutableQuantiles[] grpcQueueTimeMillisQuantiles;
+
+ @Metric("Processsing time")
+ private MutableRate grpcProcessingTime;
+
+ // There should be no getter method to avoid
+ // exposing internal representation. FindBugs error raised.
+ private MutableQuantiles[] grpcProcessingTimeMillisQuantiles;
+
+ @Metric("Number of active clients connected")
+ private MutableCounterLong numOpenClientConnections;
+
+ public void incrSentBytes(long byteCount) {
+ sentBytes.incr(byteCount);
+ }
+
+ public void incrReceivedBytes(long byteCount) {
+ receivedBytes.incr(byteCount);
+ }
+
+ public void incrUnknownMessagesSent() {
+ unknownMessagesSent.incr();
+ }
+
+ public void incrUnknownMessagesReceived() {
+ unknownMessagesReceived.incr();
+ }
+
+ public void addGrpcQueueTime(int queueTime) {
+ grpcQueueTime.add(queueTime);
+ if (grpcQuantileEnable) {
+ for (MutableQuantiles q : grpcQueueTimeMillisQuantiles) {
+ if (q != null) {
+ q.add(queueTime);
+ }
+ }
+ }
+ }
+
+ public void addGrpcProcessingTime(int processingTime) {
+ grpcProcessingTime.add(processingTime);
+ if (grpcQuantileEnable) {
+ for (MutableQuantiles q : grpcProcessingTimeMillisQuantiles) {
+ if (q != null) {
+ q.add(processingTime);
+ }
+ }
+ }
+ }
+
+ public void inrcNumOpenClientConnections() {
+ numOpenClientConnections.incr();
+ }
+
+ public void decrNumOpenClientConnections() {
+ numOpenClientConnections.incr(-1);
+ }
+
+ public long getSentBytes() {
+ return sentBytes.value();
+ }
+
+ public long getReceivedBytes() {
+ return receivedBytes.value();
+ }
+
+ public long getUnknownMessagesSent() {
+ return unknownMessagesSent.value();
+ }
+
+ public long getUnknownMessagesReceived() {
+ return unknownMessagesReceived.value();
+ }
+
+ public MutableRate getGrpcQueueTime() {
+ return grpcQueueTime;
+ }
+
+ public MutableRate getGrpcProcessingTime() {
+ return grpcProcessingTime;
+ }
+
+ public long getNumActiveClientConnections() {
+ return numOpenClientConnections.value();
+ }
+
+ public void setRequestType(String requestType) {
+ this.requestType = requestType;
+ }
+
+ public String getRequestType() {
+ return requestType;
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java
new file mode 100644
index 0000000000..a9fb7ad1f7
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerRequestInterceptor.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.grpc.metrics;
+
+import com.google.protobuf.AbstractMessage;
+import io.grpc.ForwardingServerCallListener.SimpleForwardingServerCallListener;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+
+/**
+ * Interceptor to gather metrics based on grpc server request.
+ */
+public class GrpcMetricsServerRequestInterceptor implements ServerInterceptor {
+
+ private final GrpcMetrics grpcMetrics;
+ private long receivedTime;
+ private long startTime;
+ private long endTime;
+
+ public GrpcMetricsServerRequestInterceptor(
+ GrpcMetrics grpcMetrics) {
+ super();
+ this.grpcMetrics = grpcMetrics;
+ }
+
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
+ ServerCall<ReqT, RespT> serverCall, Metadata headers,
+ ServerCallHandler<ReqT, RespT> serverCallHandler) {
+
+ // received time
+ receivedTime = System.nanoTime();
+
+ return new SimpleForwardingServerCallListener<ReqT>(
+ serverCallHandler.startCall(serverCall, headers)) {
+
+ @Override
+ public void onMessage(ReqT message) {
+ // start time
+ startTime = System.nanoTime();
+
+ long messageSize = 0;
+ if (message instanceof AbstractMessage) {
+ AbstractMessage parsedMessage = (AbstractMessage) message;
+ messageSize += parsedMessage.getSerializedSize();
+ } else {
+ grpcMetrics.incrUnknownMessagesReceived();
+ }
+
+ grpcMetrics.incrReceivedBytes(messageSize);
+
+ String[] messageFields = message.toString()
+ .split(System.lineSeparator());
+ // messageFields[0] should be in the format
+ // cmdType: "type"
+ String[] cmdTypeLine = messageFields[0].split(":");
+
+ // Get only the type and remove any leading spaces
+ grpcMetrics.setRequestType(cmdTypeLine[1].trim());
+
+ super.onMessage(message);
+ }
+
+ @Override
+ public void onComplete() {
+ super.onComplete();
+ // end time
+ endTime = System.nanoTime();
+
+ int queueTime = (int) (startTime - receivedTime);
+ int processingTime = (int) (endTime - startTime);
+
+ // set metrics queue time
+ grpcMetrics.addGrpcQueueTime(queueTime);
+
+ // set metrics processing time
+ grpcMetrics.addGrpcProcessingTime(processingTime);
+ }
+ };
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java
new file mode 100644
index 0000000000..dc342a635b
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerResponseInterceptor.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.grpc.metrics;
+
+import com.google.protobuf.AbstractMessage;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import io.grpc.ForwardingServerCall;
+
+/**
+ * Interceptor to gather metrics based on grpc server response.
+ */
+public class GrpcMetricsServerResponseInterceptor implements ServerInterceptor
{
+
+ private final GrpcMetrics grpcMetrics;
+
+ public GrpcMetricsServerResponseInterceptor(
+ GrpcMetrics grpcMetrics) {
+ super();
+ this.grpcMetrics = grpcMetrics;
+ }
+
+ @Override
+ public <ReqT, RespT> ServerCall.Listener<ReqT> interceptCall(
+ ServerCall<ReqT, RespT> serverCall, Metadata headers,
+ ServerCallHandler<ReqT, RespT> serverCallHandler) {
+
+ return serverCallHandler.startCall(
+ new ForwardingServerCall
+ .SimpleForwardingServerCall<ReqT, RespT>(serverCall) {
+ @Override
+ public void sendMessage(RespT message) {
+
+ long messageSize = 0;
+ if (message instanceof AbstractMessage) {
+ AbstractMessage parsedMessage = (AbstractMessage) message;
+ messageSize += parsedMessage.getSerializedSize();
+ } else {
+ grpcMetrics.incrUnknownMessagesSent();
+ }
+
+ grpcMetrics.incrSentBytes(messageSize);
+ super.sendMessage(message);
+ }
+ }, headers);
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java
new file mode 100644
index 0000000000..031a749d86
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/GrpcMetricsServerTransportFilter.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.grpc.metrics;
+
+import io.grpc.Attributes;
+import io.grpc.ServerTransportFilter;
+
+/**
+ * Transport filter class for tracking active client connections.
+ */
+public class GrpcMetricsServerTransportFilter extends ServerTransportFilter {
+
+ private GrpcMetrics grpcMetrics;
+
+ public GrpcMetricsServerTransportFilter(
+ GrpcMetrics grpcMetrics) {
+ super();
+ this.grpcMetrics = grpcMetrics;
+ }
+
+ @Override
+ public Attributes transportReady(Attributes transportAttrs) {
+ grpcMetrics.inrcNumOpenClientConnections();
+ return super.transportReady(transportAttrs);
+ }
+
+ @Override
+ public void transportTerminated(Attributes transportAttrs) {
+ grpcMetrics.decrNumOpenClientConnections();
+ super.transportTerminated(transportAttrs);
+ }
+}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/package-info.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/package-info.java
new file mode 100644
index 0000000000..a8f2b8ec32
--- /dev/null
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/grpc/metrics/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains classes related
+ * to gathering metrics for using gRPC.
+ */
+package org.apache.hadoop.ozone.grpc.metrics;
diff --git a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
index 6c84c30543..30a53e3714 100755
--- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh
+++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh
@@ -67,6 +67,8 @@ execute_robot_test scm -v PREFIX:${prefix} ec/read.robot
docker-compose up -d --no-recreate --scale datanode=3
execute_robot_test scm -v PREFIX:${prefix} ec/read.robot
+execute_robot_test s3g grpc/grpc-om-s3-metrics.robot
+
execute_robot_test scm snapshot
stop_docker_env
diff --git a/hadoop-ozone/dist/src/main/smoketest/grpc/grpc-om-s3-metrics.robot
b/hadoop-ozone/dist/src/main/smoketest/grpc/grpc-om-s3-metrics.robot
new file mode 100644
index 0000000000..2b0d4bfeb4
--- /dev/null
+++ b/hadoop-ozone/dist/src/main/smoketest/grpc/grpc-om-s3-metrics.robot
@@ -0,0 +1,113 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation gRPC Om S3 gateway metrics test
+Library OperatingSystem
+Library String
+Library BuiltIn
+Resource ../commonlib.robot
+Resource ../s3/commonawslib.robot
+Test Timeout 5 minutes
+Suite Setup Setup s3 tests
+
+*** Variables ***
+${OM_URL} http://${OM_SERVICE_ID}:9874
+${OM_JMX_ENDPOINT} ${OM_URL}/jmx
+${GRPC_METRICS_NAME} GrpcMetrics
+
+*** Keywords ***
+Check gRPC conf
+ ${confKey} = Execute And Ignore Error ozone getconf confKey
ozone.om.transport.class
+ ${result} = Evaluate "GrpcOmTransport" in
"""${confKey}"""
+ IF ${result} == ${True}
+ Set Suite Variable ${GRPC_ENABLED} true
+ ELSE
+ Set Suite Variable ${GRPC_ENABLED} false
+ END
+
+Verify endpoint is up
+ [arguments] ${url}
+ Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit HTTP user
+ ${result} = Execute curl --negotiate
-u : -v -s -I ${url}
+ Should contain ${result} 200 OK
+
+Get SentBytes
+ ${sentBytes} = Execute curl --negotiate -u : -LSs
${OM_JMX_ENDPOINT} | sed -n '/${GRPC_METRICS_NAME}/,/}/p' | grep 'SentBytes' |
grep -Eo '[0-9]{1,}'
+ [return] ${sentBytes}
+
+Get ReceivedBytes
+ ${receivedBytes} = Execute curl --negotiate -u : -LSs
${OM_JMX_ENDPOINT} | sed -n '/${GRPC_METRICS_NAME}/,/}/p' | grep
'ReceivedBytes' | grep -Eo '[0-9]{1,}'
+ [return] ${receivedBytes}
+
+Get NumOpenClientConnections
+ ${activeConnections} = Execute curl --negotiate -u : -LSs
${OM_JMX_ENDPOINT} | sed -n '/${GRPC_METRICS_NAME}/,/}/p' | grep
'NumOpenClientConnections' | grep -Eo '[0-9]{1,}'
+ [return] ${activeConnections}
+
+SentBytes are equal to zero
+ ${sentBytes} = Get SentBytes
+ Should be true ${sentBytes} == 0
+
+ReceivedBytes are equal to zero
+ ${receivedBytes} = Get ReceivedBytes
+ Should be true ${receivedBytes} == 0
+
+NumOpenClientConnections are equal to zero
+ ${activeConnections} = Get NumOpenClientConnections
+ Should be true ${activeConnections}
== 0
+
+SentBytes are higher than zero
+ ${sentBytes} = Get SentBytes
+ Should be true ${sentBytes} > 0
+
+ReceivedBytes are higher than zero
+ ${receivedBytes} = Get ReceivedBytes
+ Should be true ${receivedBytes} > 0
+
+NumOpenClientConnections are higher than zero
+ ${activeConnections} = Get NumOpenClientConnections
+ Should be true
${activeConnections} > 0
+
+*** Test Cases ***
+Test gRPC conf
+ Check gRPC conf
+
+Test OM JMX endpoint
+ Verify endpoint is up ${OM_JMX_ENDPOINT}
+
+Check that metrics are registered
+ ${result} = Execute curl --negotiate
-u : -LSs ${OM_JMX_ENDPOINT} | grep ${GRPC_METRICS_NAME}
+ Should contain ${result}
${GRPC_METRICS_NAME}
+
+Check bytes sent
+ IF '${GRPC_ENABLED}' == 'true'
+ Wait Until Keyword Succeeds 90sec 10sec SentBytes are
higher than zero
+ ELSE
+ SentBytes are equal to zero
+ END
+
+Check bytes received
+ IF '${GRPC_ENABLED}' == 'true'
+ Wait Until Keyword Succeeds 90sec 10sec ReceivedBytes
are higher than zero
+ ELSE
+ ReceivedBytes are equal to zero
+ END
+
+Check active connections
+ IF '${GRPC_ENABLED}' == 'true'
+ Wait Until Keyword Succeeds 90sec 10sec
NumOpenClientConnections are higher than zero
+ ELSE
+ NumOpenClientConnections are equal to zero
+ END
\ No newline at end of file
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index 6311f5775b..17de1a3c97 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -106,6 +106,7 @@ public class TestOzoneConfigurationFields extends
TestConfigurationFieldsBase {
OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_WORKERS,
OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_SCRUBBING_SERVICE_TIMEOUT,
OzoneConfigKeys.OZONE_RECOVERING_CONTAINER_TIMEOUT,
+ OzoneConfigKeys.OZONE_GPRC_METRICS_PERCENTILES_INTERVALS_KEY,
ReconConfigKeys.RECON_SCM_CONFIG_PREFIX,
ReconConfigKeys.OZONE_RECON_ADDRESS_KEY,
ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
index 085affb819..16f8af31a7 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
@@ -24,7 +24,11 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.security.ssl.KeyStoresFactory;
+import
org.apache.hadoop.ozone.grpc.metrics.GrpcMetricsServerRequestInterceptor;
+import
org.apache.hadoop.ozone.grpc.metrics.GrpcMetricsServerResponseInterceptor;
+import org.apache.hadoop.ozone.grpc.metrics.GrpcMetricsServerTransportFilter;
import org.apache.hadoop.ozone.ha.ConfUtils;
+import org.apache.hadoop.ozone.grpc.metrics.GrpcMetrics;
import
org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
import org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransport;
import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager;
@@ -35,15 +39,13 @@ import io.grpc.netty.NettyServerBuilder;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.grpc.Server;
+import io.grpc.ServerInterceptors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_PROVIDER;
-import static org.apache.hadoop.hdds.HddsConfigKeys
- .HDDS_GRPC_TLS_PROVIDER_DEFAULT;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_PROVIDER_DEFAULT;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT;
@@ -54,8 +56,9 @@ public class GrpcOzoneManagerServer {
private static final Logger LOG =
LoggerFactory.getLogger(GrpcOzoneManagerServer.class);
+ private final GrpcMetrics omS3gGrpcMetrics;
private Server server;
- private int port = 8981;
+ private int port;
private final int maxSize;
public GrpcOzoneManagerServer(OzoneConfiguration config,
@@ -79,7 +82,9 @@ public class GrpcOzoneManagerServer {
GrpcOmTransport.GrpcOmTransportConfig.class).
getPort();
}
-
+
+ this.omS3gGrpcMetrics = GrpcMetrics.create(config);
+
init(omTranslator,
delegationTokenMgr,
config,
@@ -92,9 +97,14 @@ public class GrpcOzoneManagerServer {
CertificateClient caClient) {
NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
.maxInboundMessageSize(maxSize)
- .addService(new OzoneManagerServiceGrpc(omTranslator,
- delegationTokenMgr,
- omServerConfig));
+ .addService(ServerInterceptors.intercept(
+ new OzoneManagerServiceGrpc(omTranslator,
+ delegationTokenMgr,
+ omServerConfig),
+ new GrpcMetricsServerResponseInterceptor(omS3gGrpcMetrics),
+ new GrpcMetricsServerRequestInterceptor(omS3gGrpcMetrics)))
+ .addTransportFilter(
+ new GrpcMetricsServerTransportFilter(omS3gGrpcMetrics));
SecurityConfig secConf = new SecurityConfig(omServerConfig);
if (secConf.isSecurityEnabled() && secConf.isGrpcTlsEnabled()) {
@@ -128,6 +138,8 @@ public class GrpcOzoneManagerServer {
LOG.info("Server {} is shutdown", getClass().getSimpleName());
} catch (InterruptedException ex) {
LOG.warn("{} couldn't be stopped gracefully",
getClass().getSimpleName());
+ } finally {
+ omS3gGrpcMetrics.unRegister();
}
}
public int getPort() {
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
index 5951c393b6..2faa3646e4 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerServiceGrpc.java
@@ -47,8 +47,8 @@ public class OzoneManagerServiceGrpc extends
OzoneManagerServiceImplBase {
* RpcController is not used and hence is set to null.
*/
private static final RpcController NULL_RPC_CONTROLLER = null;
- private OzoneManagerProtocolServerSideTranslatorPB omTranslator;
- private OzoneDelegationTokenSecretManager delegationTokenMgr;
+ private final OzoneManagerProtocolServerSideTranslatorPB omTranslator;
+ private final OzoneDelegationTokenSecretManager delegationTokenMgr;
private final SecurityConfig secConfig;
OzoneManagerServiceGrpc(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
