This is an automated email from the ASF dual-hosted git repository.
ferhui pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 99cb2b6 HDFS-16065. RBF: Add metrics to record Router's operations
(#3100)
99cb2b6 is described below
commit 99cb2b6b7fb8a075f40d93bafbc455966705fd70
Author: Symious <[email protected]>
AuthorDate: Thu Sep 9 18:45:23 2021 +0800
HDFS-16065. RBF: Add metrics to record Router's operations (#3100)
---
.../hdfs/server/federation/router/Router.java | 12 +
.../federation/router/RouterClientMetrics.java | 645 +++++++++++++++++++++
.../federation/router/RouterMetricsService.java | 17 +
.../server/federation/router/RouterRpcClient.java | 6 +
.../metrics/TestRouterClientMetrics.java | 174 ++++++
5 files changed, 854 insertions(+)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index ae9b62b..01b71ff 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -644,6 +644,18 @@ public class Router extends CompositeService implements
}
/**
+ * Get the metrics system for the Router Client.
+ *
+ * @return Router Client metrics.
+ */
+ public RouterClientMetrics getRouterClientMetrics() {
+ if (this.metrics != null) {
+ return this.metrics.getRouterClientMetrics();
+ }
+ return null;
+ }
+
+ /**
* Get the federation metrics.
*
* @return Federation metrics.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientMetrics.java
new file mode 100644
index 0000000..ac388194
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterClientMetrics.java
@@ -0,0 +1,645 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.router;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+
+import java.lang.reflect.Method;
+
+import static org.apache.hadoop.metrics2.impl.MsInfo.ProcessName;
+import static org.apache.hadoop.metrics2.impl.MsInfo.SessionId;
+
+/**
+ * This class is for maintaining the various Router Client activity statistics
+ * and publishing them through the metrics interfaces.
+ */
+@Metrics(name="RouterClientActivity", about="Router metrics", context="dfs")
+public class RouterClientMetrics {
+
+ private final MetricsRegistry registry = new MetricsRegistry("router");
+
+ @Metric private MutableCounterLong getBlockLocationsOps;
+ @Metric private MutableCounterLong getServerDefaultsOps;
+ @Metric private MutableCounterLong createOps;
+ @Metric private MutableCounterLong appendOps;
+ @Metric private MutableCounterLong recoverLeaseOps;
+ @Metric private MutableCounterLong setReplicationOps;
+ @Metric private MutableCounterLong setStoragePolicyOps;
+ @Metric private MutableCounterLong getStoragePoliciesOps;
+ @Metric private MutableCounterLong setPermissionOps;
+ @Metric private MutableCounterLong setOwnerOps;
+ @Metric private MutableCounterLong addBlockOps;
+ @Metric private MutableCounterLong getAdditionalDatanodeOps;
+ @Metric private MutableCounterLong abandonBlockOps;
+ @Metric private MutableCounterLong completeOps;
+ @Metric private MutableCounterLong updateBlockForPipelineOps;
+ @Metric private MutableCounterLong updatePipelineOps;
+ @Metric private MutableCounterLong getPreferredBlockSizeOps;
+ @Metric private MutableCounterLong renameOps;
+ @Metric private MutableCounterLong rename2Ops;
+ @Metric private MutableCounterLong concatOps;
+ @Metric private MutableCounterLong truncateOps;
+ @Metric private MutableCounterLong deleteOps;
+ @Metric private MutableCounterLong mkdirsOps;
+ @Metric private MutableCounterLong renewLeaseOps;
+ @Metric private MutableCounterLong getListingOps;
+ @Metric private MutableCounterLong getBatchedListingOps;
+ @Metric private MutableCounterLong getFileInfoOps;
+ @Metric private MutableCounterLong isFileClosedOps;
+ @Metric private MutableCounterLong getFileLinkInfoOps;
+ @Metric private MutableCounterLong getLocatedFileInfoOps;
+ @Metric private MutableCounterLong getStatsOps;
+ @Metric private MutableCounterLong getDatanodeReportOps;
+ @Metric private MutableCounterLong getDatanodeStorageReportOps;
+ @Metric private MutableCounterLong setSafeModeOps;
+ @Metric private MutableCounterLong restoreFailedStorageOps;
+ @Metric private MutableCounterLong saveNamespaceOps;
+ @Metric private MutableCounterLong rollEditsOps;
+ @Metric private MutableCounterLong refreshNodesOps;
+ @Metric private MutableCounterLong finalizeUpgradeOps;
+ @Metric private MutableCounterLong upgradeStatusOps;
+ @Metric private MutableCounterLong rollingUpgradeOps;
+ @Metric private MutableCounterLong metaSaveOps;
+ @Metric private MutableCounterLong listCorruptFileBlocksOps;
+ @Metric private MutableCounterLong setBalancerBandwidthOps;
+ @Metric private MutableCounterLong getContentSummaryOps;
+ @Metric private MutableCounterLong fsyncOps;
+ @Metric private MutableCounterLong setTimesOps;
+ @Metric private MutableCounterLong createSymlinkOps;
+ @Metric private MutableCounterLong getLinkTargetOps;
+ @Metric private MutableCounterLong allowSnapshotOps;
+ @Metric private MutableCounterLong disallowSnapshotOps;
+ @Metric private MutableCounterLong renameSnapshotOps;
+ @Metric private MutableCounterLong getSnapshottableDirListingOps;
+ @Metric private MutableCounterLong getSnapshotListingOps;
+ @Metric private MutableCounterLong getSnapshotDiffReportOps;
+ @Metric private MutableCounterLong getSnapshotDiffReportListingOps;
+ @Metric private MutableCounterLong addCacheDirectiveOps;
+ @Metric private MutableCounterLong modifyCacheDirectiveOps;
+ @Metric private MutableCounterLong removeCacheDirectiveOps;
+ @Metric private MutableCounterLong listCacheDirectivesOps;
+ @Metric private MutableCounterLong addCachePoolOps;
+ @Metric private MutableCounterLong modifyCachePoolOps;
+ @Metric private MutableCounterLong removeCachePoolOps;
+ @Metric private MutableCounterLong listCachePoolsOps;
+ @Metric private MutableCounterLong modifyAclEntriesOps;
+ @Metric private MutableCounterLong removeAclEntriesOps;
+ @Metric private MutableCounterLong removeDefaultAclOps;
+ @Metric private MutableCounterLong removeAclOps;
+ @Metric private MutableCounterLong setAclOps;
+ @Metric private MutableCounterLong getAclStatusOps;
+ @Metric private MutableCounterLong createEncryptionZoneOps;
+ @Metric private MutableCounterLong getEZForPathOps;
+ @Metric private MutableCounterLong listEncryptionZonesOps;
+ @Metric private MutableCounterLong reencryptEncryptionZoneOps;
+ @Metric private MutableCounterLong listReencryptionStatusOps;
+ @Metric private MutableCounterLong setXAttrOps;
+ @Metric private MutableCounterLong getXAttrsOps;
+ @Metric private MutableCounterLong listXAttrsOps;
+ @Metric private MutableCounterLong removeXAttrsOps;
+ @Metric private MutableCounterLong checkAccessOps;
+ @Metric private MutableCounterLong getCurrentEditLogTxidOps;
+ @Metric private MutableCounterLong getEditsFromTxidOps;
+ @Metric private MutableCounterLong getDataEncryptionKeyOps;
+ @Metric private MutableCounterLong createSnapshotOps;
+ @Metric private MutableCounterLong deleteSnapshotOps;
+ @Metric private MutableCounterLong setQuotaOps;
+ @Metric private MutableCounterLong getQuotaUsageOps;
+ @Metric private MutableCounterLong reportBadBlocksOps;
+ @Metric private MutableCounterLong unsetStoragePolicyOps;
+ @Metric private MutableCounterLong getStoragePolicyOps;
+ @Metric private MutableCounterLong getErasureCodingPoliciesOps;
+ @Metric private MutableCounterLong getErasureCodingCodecsOps;
+ @Metric private MutableCounterLong addErasureCodingPoliciesOps;
+ @Metric private MutableCounterLong removeErasureCodingPolicyOps;
+ @Metric private MutableCounterLong disableErasureCodingPolicyOps;
+ @Metric private MutableCounterLong enableErasureCodingPolicyOps;
+ @Metric private MutableCounterLong getErasureCodingPolicyOps;
+ @Metric private MutableCounterLong setErasureCodingPolicyOps;
+ @Metric private MutableCounterLong unsetErasureCodingPolicyOps;
+ @Metric private MutableCounterLong getECTopologyResultForPoliciesOps;
+ @Metric private MutableCounterLong getECBlockGroupStatsOps;
+ @Metric private MutableCounterLong getReplicatedBlockStatsOps;
+ @Metric private MutableCounterLong listOpenFilesOps;
+ @Metric private MutableCounterLong msyncOps;
+ @Metric private MutableCounterLong satisfyStoragePolicyOps;
+ @Metric private MutableCounterLong getHAServiceStateOps;
+ @Metric private MutableCounterLong otherOps;
+
+ // private ConcurrentOps
+ @Metric private MutableCounterLong concurrentSetReplicationOps;
+ @Metric private MutableCounterLong concurrentSetPermissionOps;
+ @Metric private MutableCounterLong concurrentSetOwnerOps;
+ @Metric private MutableCounterLong concurrentRenameOps;
+ @Metric private MutableCounterLong concurrentRename2Ops;
+ @Metric private MutableCounterLong concurrentDeleteOps;
+ @Metric private MutableCounterLong concurrentMkdirsOps;
+ @Metric private MutableCounterLong concurrentRenewLeaseOps;
+ @Metric private MutableCounterLong concurrentGetListingOps;
+ @Metric private MutableCounterLong concurrentGetFileInfoOps;
+ @Metric private MutableCounterLong concurrentGetStatsOps;
+ @Metric private MutableCounterLong concurrentGetDatanodeReportOps;
+ @Metric private MutableCounterLong concurrentSetSafeModeOps;
+ @Metric private MutableCounterLong concurrentRestoreFailedStorageOps;
+ @Metric private MutableCounterLong concurrentSaveNamespaceOps;
+ @Metric private MutableCounterLong concurrentRollEditsOps;
+ @Metric private MutableCounterLong concurrentRefreshNodesOps;
+ @Metric private MutableCounterLong concurrentFinalizeUpgradeOps;
+ @Metric private MutableCounterLong concurrentRollingUpgradeOps;
+ @Metric private MutableCounterLong concurrentMetaSaveOps;
+ @Metric private MutableCounterLong concurrentListCorruptFileBlocksOps;
+ @Metric private MutableCounterLong concurrentSetBalancerBandwidthOps;
+ @Metric private MutableCounterLong concurrentGetContentSummaryOps;
+ @Metric private MutableCounterLong concurrentModifyAclEntriesOps;
+ @Metric private MutableCounterLong concurrentRemoveAclEntriesOps;
+ @Metric private MutableCounterLong concurrentRemoveDefaultAclOps;
+ @Metric private MutableCounterLong concurrentRemoveAclOps;
+ @Metric private MutableCounterLong concurrentSetAclOps;
+ @Metric private MutableCounterLong concurrentSetXAttrOps;
+ @Metric private MutableCounterLong concurrentRemoveXAttrOps;
+ @Metric private MutableCounterLong concurrentGetCurrentEditLogTxidOps;
+ @Metric private MutableCounterLong concurrentGetReplicatedBlockStatsOps;
+ @Metric private MutableCounterLong concurrentSetQuotaOps;
+ @Metric private MutableCounterLong concurrentGetQuotaUsageOps;
+ @Metric private MutableCounterLong concurrentOtherOps;
+
+ RouterClientMetrics(String processName, String sessionId) {
+ registry.tag(ProcessName, processName).tag(SessionId, sessionId);
+ }
+
+ public static RouterClientMetrics create(Configuration conf) {
+ String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
+ String processName = "Router";
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+
+ return ms.register(new RouterClientMetrics(processName, sessionId));
+ }
+
+ public void shutdown() {
+ DefaultMetricsSystem.shutdown();
+ }
+
+ /**
+ * Increase the metrics based on the method being invoked.
+ * @param method method being invoked
+ */
+ public void incInvokedMethod(Method method) {
+ switch (method.getName()) {
+ case "getBlockLocations":
+ getBlockLocationsOps.incr();
+ break;
+ case "getServerDefaults":
+ getServerDefaultsOps.incr();
+ break;
+ case "create":
+ createOps.incr();
+ break;
+ case "append":
+ appendOps.incr();
+ break;
+ case "recoverLease":
+ recoverLeaseOps.incr();
+ break;
+ case "setReplication":
+ setReplicationOps.incr();
+ break;
+ case "setStoragePolicy":
+ setStoragePolicyOps.incr();
+ break;
+ case "getStoragePolicies":
+ getStoragePoliciesOps.incr();
+ break;
+ case "setPermission":
+ setPermissionOps.incr();
+ break;
+ case "setOwner":
+ setOwnerOps.incr();
+ break;
+ case "addBlock":
+ addBlockOps.incr();
+ break;
+ case "getAdditionalDatanode":
+ getAdditionalDatanodeOps.incr();
+ break;
+ case "abandonBlock":
+ abandonBlockOps.incr();
+ break;
+ case "complete":
+ completeOps.incr();
+ break;
+ case "updateBlockForPipeline":
+ updateBlockForPipelineOps.incr();
+ break;
+ case "updatePipeline":
+ updatePipelineOps.incr();
+ break;
+ case "getPreferredBlockSize":
+ getPreferredBlockSizeOps.incr();
+ break;
+ case "rename":
+ renameOps.incr();
+ break;
+ case "rename2":
+ rename2Ops.incr();
+ break;
+ case "concat":
+ concatOps.incr();
+ break;
+ case "truncate":
+ truncateOps.incr();
+ break;
+ case "delete":
+ deleteOps.incr();
+ break;
+ case "mkdirs":
+ mkdirsOps.incr();
+ break;
+ case "renewLease":
+ renewLeaseOps.incr();
+ break;
+ case "getListing":
+ getListingOps.incr();
+ break;
+ case "getBatchedListing":
+ getBatchedListingOps.incr();
+ break;
+ case "getFileInfo":
+ getFileInfoOps.incr();
+ break;
+ case "isFileClosed":
+ isFileClosedOps.incr();
+ break;
+ case "getFileLinkInfo":
+ getFileLinkInfoOps.incr();
+ break;
+ case "getLocatedFileInfo":
+ getLocatedFileInfoOps.incr();
+ break;
+ case "getStats":
+ getStatsOps.incr();
+ break;
+ case "getDatanodeReport":
+ getDatanodeReportOps.incr();
+ break;
+ case "getDatanodeStorageReport":
+ getDatanodeStorageReportOps.incr();
+ break;
+ case "setSafeMode":
+ setSafeModeOps.incr();
+ break;
+ case "restoreFailedStorage":
+ restoreFailedStorageOps.incr();
+ break;
+ case "saveNamespace":
+ saveNamespaceOps.incr();
+ break;
+ case "rollEdits":
+ rollEditsOps.incr();
+ break;
+ case "refreshNodes":
+ refreshNodesOps.incr();
+ break;
+ case "finalizeUpgrade":
+ finalizeUpgradeOps.incr();
+ break;
+ case "upgradeStatus":
+ upgradeStatusOps.incr();
+ break;
+ case "rollingUpgrade":
+ rollingUpgradeOps.incr();
+ break;
+ case "metaSave":
+ metaSaveOps.incr();
+ break;
+ case "listCorruptFileBlocks":
+ listCorruptFileBlocksOps.incr();
+ break;
+ case "setBalancerBandwidth":
+ setBalancerBandwidthOps.incr();
+ break;
+ case "getContentSummary":
+ getContentSummaryOps.incr();
+ break;
+ case "fsync":
+ fsyncOps.incr();
+ break;
+ case "setTimes":
+ setTimesOps.incr();
+ break;
+ case "createSymlink":
+ createSymlinkOps.incr();
+ break;
+ case "getLinkTarget":
+ getLinkTargetOps.incr();
+ break;
+ case "allowSnapshot":
+ allowSnapshotOps.incr();
+ break;
+ case "disallowSnapshot":
+ disallowSnapshotOps.incr();
+ break;
+ case "renameSnapshot":
+ renameSnapshotOps.incr();
+ break;
+ case "getSnapshottableDirListing":
+ getSnapshottableDirListingOps.incr();
+ break;
+ case "getSnapshotListing":
+ getSnapshotListingOps.incr();
+ break;
+ case "getSnapshotDiffReport":
+ getSnapshotDiffReportOps.incr();
+ break;
+ case "getSnapshotDiffReportListing":
+ getSnapshotDiffReportListingOps.incr();
+ break;
+ case "addCacheDirective":
+ addCacheDirectiveOps.incr();
+ break;
+ case "modifyCacheDirective":
+ modifyCacheDirectiveOps.incr();
+ break;
+ case "removeCacheDirective":
+ removeCacheDirectiveOps.incr();
+ break;
+ case "listCacheDirectives":
+ listCacheDirectivesOps.incr();
+ break;
+ case "addCachePool":
+ addCachePoolOps.incr();
+ break;
+ case "modifyCachePool":
+ modifyCachePoolOps.incr();
+ break;
+ case "removeCachePool":
+ removeCachePoolOps.incr();
+ break;
+ case "listCachePools":
+ listCachePoolsOps.incr();
+ break;
+ case "modifyAclEntries":
+ modifyAclEntriesOps.incr();
+ break;
+ case "removeAclEntries":
+ removeAclEntriesOps.incr();
+ break;
+ case "removeDefaultAcl":
+ removeDefaultAclOps.incr();
+ break;
+ case "removeAcl":
+ removeAclOps.incr();
+ break;
+ case "setAcl":
+ setAclOps.incr();
+ break;
+ case "getAclStatus":
+ getAclStatusOps.incr();
+ break;
+ case "createEncryptionZone":
+ createEncryptionZoneOps.incr();
+ break;
+ case "getEZForPath":
+ getEZForPathOps.incr();
+ break;
+ case "listEncryptionZones":
+ listEncryptionZonesOps.incr();
+ break;
+ case "reencryptEncryptionZone":
+ reencryptEncryptionZoneOps.incr();
+ break;
+ case "listReencryptionStatus":
+ listReencryptionStatusOps.incr();
+ break;
+ case "setXAttr":
+ setXAttrOps.incr();
+ break;
+ case "getXAttrs":
+ getXAttrsOps.incr();
+ break;
+ case "listXAttrs":
+ listXAttrsOps.incr();
+ break;
+ case "removeXAttr":
+ removeXAttrsOps.incr();
+ break;
+ case "checkAccess":
+ checkAccessOps.incr();
+ break;
+ case "getCurrentEditLogTxid":
+ getCurrentEditLogTxidOps.incr();
+ break;
+ case "getEditsFromTxid":
+ getEditsFromTxidOps.incr();
+ break;
+ case "getDataEncryptionKey":
+ getDataEncryptionKeyOps.incr();
+ break;
+ case "createSnapshot":
+ createSnapshotOps.incr();
+ break;
+ case "deleteSnapshot":
+ deleteSnapshotOps.incr();
+ break;
+ case "setQuota":
+ setQuotaOps.incr();
+ break;
+ case "getQuotaUsage":
+ getQuotaUsageOps.incr();
+ break;
+ case "reportBadBlocks":
+ reportBadBlocksOps.incr();
+ break;
+ case "unsetStoragePolicy":
+ unsetStoragePolicyOps.incr();
+ break;
+ case "getStoragePolicy":
+ getStoragePolicyOps.incr();
+ break;
+ case "getErasureCodingPolicies":
+ getErasureCodingPoliciesOps.incr();
+ break;
+ case "getErasureCodingCodecs":
+ getErasureCodingCodecsOps.incr();
+ break;
+ case "addErasureCodingPolicies":
+ addErasureCodingPoliciesOps.incr();
+ break;
+ case "removeErasureCodingPolicy":
+ removeErasureCodingPolicyOps.incr();
+ break;
+ case "disableErasureCodingPolicy":
+ disableErasureCodingPolicyOps.incr();
+ break;
+ case "enableErasureCodingPolicy":
+ enableErasureCodingPolicyOps.incr();
+ break;
+ case "getErasureCodingPolicy":
+ getErasureCodingPolicyOps.incr();
+ break;
+ case "setErasureCodingPolicy":
+ setErasureCodingPolicyOps.incr();
+ break;
+ case "unsetErasureCodingPolicy":
+ unsetErasureCodingPolicyOps.incr();
+ break;
+ case "getECTopologyResultForPolicies":
+ getECTopologyResultForPoliciesOps.incr();
+ break;
+ case "getECBlockGroupStats":
+ getECBlockGroupStatsOps.incr();
+ break;
+ case "getReplicatedBlockStats":
+ getReplicatedBlockStatsOps.incr();
+ break;
+ case "listOpenFiles":
+ listOpenFilesOps.incr();
+ break;
+ case "msync":
+ msyncOps.incr();
+ break;
+ case "satisfyStoragePolicy":
+ satisfyStoragePolicyOps.incr();
+ break;
+ case "getHAServiceState":
+ getHAServiceStateOps.incr();
+ break;
+ default:
+ otherOps.incr();
+ }
+ }
+
+ /**
+ * Increase the concurrent metrics based on the method being invoked.
+ * @param method concurrently invoked method
+ */
+ public void incInvokedConcurrent(Method method){
+ switch (method.getName()) {
+ case "setReplication":
+ concurrentSetReplicationOps.incr();
+ break;
+ case "setPermission":
+ concurrentSetPermissionOps.incr();
+ break;
+ case "setOwner":
+ concurrentSetOwnerOps.incr();
+ break;
+ case "rename":
+ concurrentRenameOps.incr();
+ break;
+ case "rename2":
+ concurrentRename2Ops.incr();
+ break;
+ case "delete":
+ concurrentDeleteOps.incr();
+ break;
+ case "mkdirs":
+ concurrentMkdirsOps.incr();
+ break;
+ case "renewLease":
+ concurrentRenewLeaseOps.incr();
+ break;
+ case "getListing":
+ concurrentGetListingOps.incr();
+ break;
+ case "getFileInfo":
+ concurrentGetFileInfoOps.incr();
+ break;
+ case "getStats":
+ concurrentGetStatsOps.incr();
+ break;
+ case "getDatanodeReport":
+ concurrentGetDatanodeReportOps.incr();
+ break;
+ case "setSafeMode":
+ concurrentSetSafeModeOps.incr();
+ break;
+ case "restoreFailedStorage":
+ concurrentRestoreFailedStorageOps.incr();
+ break;
+ case "saveNamespace":
+ concurrentSaveNamespaceOps.incr();
+ break;
+ case "rollEdits":
+ concurrentRollEditsOps.incr();
+ break;
+ case "refreshNodes":
+ concurrentRefreshNodesOps.incr();
+ break;
+ case "finalizeUpgrade":
+ concurrentFinalizeUpgradeOps.incr();
+ break;
+ case "rollingUpgrade":
+ concurrentRollingUpgradeOps.incr();
+ break;
+ case "metaSave":
+ concurrentMetaSaveOps.incr();
+ break;
+ case "listCorruptFileBlocks":
+ concurrentListCorruptFileBlocksOps.incr();
+ break;
+ case "setBalancerBandwidth":
+ concurrentSetBalancerBandwidthOps.incr();
+ break;
+ case "getContentSummary":
+ concurrentGetContentSummaryOps.incr();
+ break;
+ case "modifyAclEntries":
+ concurrentModifyAclEntriesOps.incr();
+ break;
+ case "removeAclEntries":
+ concurrentRemoveAclEntriesOps.incr();
+ break;
+ case "removeDefaultAcl":
+ concurrentRemoveDefaultAclOps.incr();
+ break;
+ case "removeAcl":
+ concurrentRemoveAclOps.incr();
+ break;
+ case "setAcl":
+ concurrentSetAclOps.incr();
+ break;
+ case "setXAttr":
+ concurrentSetXAttrOps.incr();
+ break;
+ case "removeXAttr":
+ concurrentRemoveXAttrOps.incr();
+ break;
+ case "getCurrentEditLogTxid":
+ concurrentGetCurrentEditLogTxidOps.incr();
+ break;
+ case "getReplicatedBlockStats":
+ concurrentGetReplicatedBlockStatsOps.incr();
+ break;
+ case "setQuota":
+ concurrentSetQuotaOps.incr();
+ break;
+ case "getQuotaUsage":
+ concurrentGetQuotaUsageOps.incr();
+ break;
+ default :
+ concurrentOtherOps.incr();
+ }
+ }
+}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
index 1ed15bf..fd0ebbc 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
@@ -33,6 +33,8 @@ public class RouterMetricsService extends AbstractService {
/** Router metrics. */
private RouterMetrics routerMetrics;
+ /** Router Client metrics. */
+ private RouterClientMetrics routerClientMetrics;
/** Federation metrics. */
private RBFMetrics rbfMetrics;
/** Namenode mock metrics. */
@@ -47,6 +49,7 @@ public class RouterMetricsService extends AbstractService {
@Override
protected void serviceInit(Configuration configuration) throws Exception {
this.routerMetrics = RouterMetrics.create(configuration);
+ this.routerClientMetrics = RouterClientMetrics.create(configuration);
}
@Override
@@ -74,6 +77,11 @@ public class RouterMetricsService extends AbstractService {
if (this.routerMetrics != null) {
this.routerMetrics.shutdown();
}
+
+ // Shutdown client metrics
+ if (this.routerClientMetrics != null) {
+ this.routerClientMetrics.shutdown();
+ }
}
/**
@@ -86,6 +94,15 @@ public class RouterMetricsService extends AbstractService {
}
/**
+ * Get the metrics system for the Router Client.
+ *
+ * @return Router Client metrics.
+ */
+ public RouterClientMetrics getRouterClientMetrics() {
+ return this.routerClientMetrics;
+ }
+
+ /**
* Get the federation metrics.
*
* @return Federation metrics.
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
index a4833ff..d8eb5f1 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcClient.java
@@ -471,6 +471,9 @@ public class RouterRpcClient {
if (this.rpcMonitor != null) {
this.rpcMonitor.proxyOpComplete(true);
}
+ if (this.router.getRouterClientMetrics() != null) {
+ this.router.getRouterClientMetrics().incInvokedMethod(method);
+ }
return ret;
} catch (IOException ioe) {
ioes.put(namenode, ioe);
@@ -1386,6 +1389,9 @@ public class RouterRpcClient {
if (rpcMonitor != null) {
rpcMonitor.proxyOp();
}
+ if (this.router.getRouterClientMetrics() != null) {
+ this.router.getRouterClientMetrics().incInvokedConcurrent(m);
+ }
acquirePermit(CONCURRENT_NS, ugi, method);
try {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java
new file mode 100644
index 0000000..da16c05
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/metrics/TestRouterClientMetrics.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.federation.metrics;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
+import org.apache.hadoop.hdfs.server.federation.MockResolver;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.router.Router;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+
+/**
+ * Test case for RouterClientMetrics.
+ */
+public class TestRouterClientMetrics {
+ private static final Configuration CONF = new HdfsConfiguration();
+ private static final String ROUTER_METRICS = "RouterClientActivity";
+ static {
+ CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 100);
+ CONF.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, 1);
+ CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1);
+ }
+
+ private static final int NUM_SUBCLUSTERS = 2;
+ private static final int NUM_DNS = 3;
+
+
+ /** Federated HDFS cluster. */
+ private static MiniRouterDFSCluster cluster;
+
+ /** The first Router Context for this federated cluster. */
+ private MiniRouterDFSCluster.RouterContext routerContext;
+
+ /** The first Router for this federated cluster. */
+ private Router router;
+
+ /** Filesystem interface to the Router. */
+ private FileSystem routerFS;
+ /** Filesystem interface to the Namenode. */
+ private FileSystem nnFS;
+
+ @BeforeClass
+ public static void globalSetUp() throws Exception {
+ cluster = new MiniRouterDFSCluster(false, NUM_SUBCLUSTERS);
+ cluster.setNumDatanodesPerNameservice(NUM_DNS);
+ cluster.startCluster();
+
+ Configuration routerConf = new RouterConfigBuilder()
+ .metrics()
+ .rpc()
+ .quota()
+ .build();
+ cluster.addRouterOverrides(routerConf);
+ cluster.startRouters();
+
+ // Register and verify all NNs with all routers
+ cluster.registerNamenodes();
+ cluster.waitNamenodeRegistration();
+
+ }
+
+ @Before
+ public void testSetup() throws Exception {
+ // Create mock locations
+ cluster.installMockLocations();
+
+ // Delete all files via the NNs and verify
+ cluster.deleteAllFiles();
+
+ // Create test fixtures on NN
+ cluster.createTestDirectoriesNamenode();
+
+ // Wait to ensure NN has fully created its test directories
+ Thread.sleep(100);
+
+ routerContext = cluster.getRouters().get(0);
+ this.routerFS = routerContext.getFileSystem();
+
+ // Add extra location to the root mount / such that the root mount points:
+ // /
+ // ns0 -> /
+ // ns1 -> /
+ router = routerContext.getRouter();
+ MockResolver resolver = (MockResolver) router.getSubclusterResolver();
+ resolver.addLocation("/", cluster.getNameservices().get(1), "/");
+
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ cluster.shutdown();
+ }
+
+ @Test
+ public void testGetListing() throws IOException {
+ routerFS.listStatus(new Path("/"));
+ assertCounter("GetListingOps", 2L, getMetrics(ROUTER_METRICS));
+ assertCounter("ConcurrentGetListingOps", 1L, getMetrics(ROUTER_METRICS));
+ }
+
+ @Test
+ public void testCreate() throws IOException {
+ Path testFile = new Path("/testCreate");
+ routerFS.create(testFile);
+ assertCounter("CreateOps", 1L, getMetrics(ROUTER_METRICS));
+ }
+
+ @Test
+ public void testGetServerDefaults() throws IOException {
+ router.getRpcServer().getServerDefaults();
+ assertCounter("GetServerDefaultsOps", 1L, getMetrics(ROUTER_METRICS));
+ }
+
+ @Test
+ public void testSetQuota() throws Exception {
+ router.getRpcServer().setQuota("/", 1L, 1L, null);
+ assertCounter("SetQuotaOps", 2L, getMetrics(ROUTER_METRICS));
+ assertCounter("ConcurrentSetQuotaOps", 1L, getMetrics(ROUTER_METRICS));
+ }
+
+ @Test
+ public void testGetQuota() throws Exception {
+ router.getRpcServer().getQuotaUsage("/");
+ assertCounter("GetQuotaUsageOps", 2L, getMetrics(ROUTER_METRICS));
+ assertCounter("ConcurrentGetQuotaUsageOps", 1L,
getMetrics(ROUTER_METRICS));
+ }
+
+ @Test
+ public void testRenewLease() throws Exception {
+ router.getRpcServer().renewLease("test");
+ assertCounter("RenewLeaseOps", 2L, getMetrics(ROUTER_METRICS));
+ assertCounter("ConcurrentRenewLeaseOps", 1L, getMetrics(ROUTER_METRICS));
+ }
+
+ @Test
+ public void testGetDatanodeReport() throws Exception {
+ router.getRpcServer().
+ getDatanodeReport(HdfsConstants.DatanodeReportType.LIVE);
+ assertCounter("GetDatanodeReportOps", 2L, getMetrics(ROUTER_METRICS));
+ assertCounter("ConcurrentGetDatanodeReportOps", 1L,
+ getMetrics(ROUTER_METRICS));
+ }
+
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
