[
https://issues.apache.org/jira/browse/HADOOP-18167?focusedWorklogId=757235&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-757235
]
ASF GitHub Bot logged work on HADOOP-18167:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 14/Apr/22 22:12
Start Date: 14/Apr/22 22:12
Worklog Time Spent: 10m
Work Description: goiri commented on code in PR #4092:
URL: https://github.com/apache/hadoop/pull/4092#discussion_r850843618
##########
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java:
##########
@@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() {
addTokenForOwnerStats(id);
}
}
+
+ /**
+ * DelegationTokenSecretManagerMetrics tracks token management operations
+ * and publishes them through the metrics interfaces.
+ */
+ @Metrics(about="Delegation token secret manager metrics", context="token")
+ static class DelegationTokenSecretManagerMetrics implements
IOStatisticsSource {
+ private static final Logger LOG =
LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class);
+
+ final static String STORE_TOKEN_STAT = "storeToken";
+ final static String UPDATE_TOKEN_STAT = "updateToken";
+ final static String REMOVE_TOKEN_STAT = "removeToken";
+ final static String TOKEN_FAILURE_STAT = "tokenFailure";
+
+ final MetricsRegistry registry;
+ final IOStatisticsStore ioStatistics;
+
+ @Metric("Rate of storage of delegation tokens and latency (milliseconds)")
+ MutableRate storeToken;
+ @Metric("Rate of update of delegation tokens and latency (milliseconds)")
+ MutableRate updateToken;
+ @Metric("Rate of removal of delegation tokens and latency (milliseconds)")
+ MutableRate removeToken;
+ @Metric("Counter of delegation tokens operation failures")
+ MutableCounterLong tokenFailure;
+
+ static DelegationTokenSecretManagerMetrics create() {
+ return DefaultMetricsSystem.instance().register(new
DelegationTokenSecretManagerMetrics());
+ }
+
+ public DelegationTokenSecretManagerMetrics() {
+ ioStatistics = iostatisticsStore()
+ .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT,
REMOVE_TOKEN_STAT)
+ .withCounters(TOKEN_FAILURE_STAT)
+ .build();
+ registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
+ LOG.debug("Initialized {}", registry);
+ }
+
+ public void addStoreToken(long value) {
Review Comment:
addTimeStoreToken()
##########
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java:
##########
@@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() {
addTokenForOwnerStats(id);
}
}
+
+ /**
+ * DelegationTokenSecretManagerMetrics tracks token management operations
+ * and publishes them through the metrics interfaces.
+ */
+ @Metrics(about="Delegation token secret manager metrics", context="token")
+ static class DelegationTokenSecretManagerMetrics implements
IOStatisticsSource {
+ private static final Logger LOG =
LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class);
+
+ final static String STORE_TOKEN_STAT = "storeToken";
+ final static String UPDATE_TOKEN_STAT = "updateToken";
+ final static String REMOVE_TOKEN_STAT = "removeToken";
+ final static String TOKEN_FAILURE_STAT = "tokenFailure";
+
+ final MetricsRegistry registry;
+ final IOStatisticsStore ioStatistics;
+
+ @Metric("Rate of storage of delegation tokens and latency (milliseconds)")
+ MutableRate storeToken;
+ @Metric("Rate of update of delegation tokens and latency (milliseconds)")
+ MutableRate updateToken;
+ @Metric("Rate of removal of delegation tokens and latency (milliseconds)")
+ MutableRate removeToken;
+ @Metric("Counter of delegation tokens operation failures")
+ MutableCounterLong tokenFailure;
+
+ static DelegationTokenSecretManagerMetrics create() {
+ return DefaultMetricsSystem.instance().register(new
DelegationTokenSecretManagerMetrics());
+ }
+
+ public DelegationTokenSecretManagerMetrics() {
+ ioStatistics = iostatisticsStore()
+ .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT,
REMOVE_TOKEN_STAT)
+ .withCounters(TOKEN_FAILURE_STAT)
+ .build();
+ registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
+ LOG.debug("Initialized {}", registry);
+ }
+
+ public void addStoreToken(long value) {
+ storeToken.add(value);
+ ioStatistics.addTimedOperation(STORE_TOKEN_STAT, value);
+ }
+
+ public void addUpdateToken(long value) {
Review Comment:
addTimeUpdateToken() and same for the rest.
##########
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java:
##########
@@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() {
addTokenForOwnerStats(id);
}
}
+
+ /**
+ * DelegationTokenSecretManagerMetrics tracks token management operations
+ * and publishes them through the metrics interfaces.
+ */
+ @Metrics(about="Delegation token secret manager metrics", context="token")
+ static class DelegationTokenSecretManagerMetrics implements
IOStatisticsSource {
+ private static final Logger LOG =
LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class);
+
+ final static String STORE_TOKEN_STAT = "storeToken";
+ final static String UPDATE_TOKEN_STAT = "updateToken";
+ final static String REMOVE_TOKEN_STAT = "removeToken";
+ final static String TOKEN_FAILURE_STAT = "tokenFailure";
+
+ final MetricsRegistry registry;
+ final IOStatisticsStore ioStatistics;
+
+ @Metric("Rate of storage of delegation tokens and latency (milliseconds)")
+ MutableRate storeToken;
+ @Metric("Rate of update of delegation tokens and latency (milliseconds)")
+ MutableRate updateToken;
+ @Metric("Rate of removal of delegation tokens and latency (milliseconds)")
+ MutableRate removeToken;
+ @Metric("Counter of delegation tokens operation failures")
+ MutableCounterLong tokenFailure;
+
+ static DelegationTokenSecretManagerMetrics create() {
+ return DefaultMetricsSystem.instance().register(new
DelegationTokenSecretManagerMetrics());
+ }
+
+ public DelegationTokenSecretManagerMetrics() {
+ ioStatistics = iostatisticsStore()
+ .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT,
REMOVE_TOKEN_STAT)
+ .withCounters(TOKEN_FAILURE_STAT)
+ .build();
+ registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
+ LOG.debug("Initialized {}", registry);
+ }
+
+ public void addStoreToken(long value) {
Review Comment:
Make the argument final too.
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java:
##########
@@ -579,4 +615,65 @@ public void testEmptyToken() throws IOException {
assertEquals(token1, token2);
assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString());
}
+
+ @Test
+ public void testDelegationTokenSecretManagerMetrics() throws Exception {
+ TestDelegationTokenSecretManager dtSecretManager =
+ new TestDelegationTokenSecretManager(24*60*60*1000,
+ 10*1000,1*1000,3600000);
+ try {
+ dtSecretManager.startThreads();
+
+ Assert.assertEquals(0,
dtSecretManager.metrics.storeToken.lastStat().numSamples());
+ final Token<TestDelegationTokenIdentifier> token =
+ generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
+ Assert.assertEquals(1,
dtSecretManager.metrics.storeToken.lastStat().numSamples());
+
+ Assert.assertEquals(0,
dtSecretManager.metrics.updateToken.lastStat().numSamples());
+ dtSecretManager.renewToken(token, "JobTracker");
+ Assert.assertEquals(1,
dtSecretManager.metrics.updateToken.lastStat().numSamples());
+
+ Assert.assertEquals(0,
dtSecretManager.metrics.removeToken.lastStat().numSamples());
+ dtSecretManager.cancelToken(token, "JobTracker");
+ Assert.assertEquals(1,
dtSecretManager.metrics.removeToken.lastStat().numSamples());
+ } finally {
+ dtSecretManager.stopThreads();
+ }
+ }
+
+ @Test
+ public void testDelegationTokenSecretManagerMetricsFailures() throws
Exception {
+ TestFailureDelegationTokenSecretManager dtSecretManager = new
TestFailureDelegationTokenSecretManager();
+
+ try {
+ dtSecretManager.startThreads();
+
+ final Token<TestDelegationTokenIdentifier> token =
+ generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
+
+ dtSecretManager.setThrowError(true);
+
+ Assert.assertEquals(0, dtSecretManager.metrics.tokenFailure.value());
+ generateDelegationToken(dtSecretManager, "SomeUser", "JobTracker");
+ Assert.assertEquals(1, dtSecretManager.metrics.tokenFailure.value());
+
+ try {
+ dtSecretManager.renewToken(token, "JobTracker");
+ Assert.fail("Expected exception");
+ } catch (Exception ex) {
+ // Expected exception
+ }
+ Assert.assertEquals(2, dtSecretManager.metrics.tokenFailure.value());
+
+ try {
+ dtSecretManager.cancelToken(token, "JobTracker");
+ Assert.fail("Expected exception");
Review Comment:
use LamdaTestUtils#intercept
##########
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java:
##########
@@ -50,6 +59,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.statistics.impl.IOStatisticsBinding.*;
Review Comment:
Better to expand this or to use the full name.
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java:
##########
@@ -579,4 +615,65 @@ public void testEmptyToken() throws IOException {
assertEquals(token1, token2);
assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString());
}
+
+ @Test
+ public void testDelegationTokenSecretManagerMetrics() throws Exception {
+ TestDelegationTokenSecretManager dtSecretManager =
+ new TestDelegationTokenSecretManager(24*60*60*1000,
+ 10*1000,1*1000,3600000);
Review Comment:
spaces after the commas
##########
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java:
##########
@@ -825,4 +859,68 @@ protected void syncTokenOwnerStats() {
addTokenForOwnerStats(id);
}
}
+
+ /**
+ * DelegationTokenSecretManagerMetrics tracks token management operations
+ * and publishes them through the metrics interfaces.
+ */
+ @Metrics(about="Delegation token secret manager metrics", context="token")
+ static class DelegationTokenSecretManagerMetrics implements
IOStatisticsSource {
+ private static final Logger LOG =
LoggerFactory.getLogger(DelegationTokenSecretManagerMetrics.class);
+
+ final static String STORE_TOKEN_STAT = "storeToken";
+ final static String UPDATE_TOKEN_STAT = "updateToken";
+ final static String REMOVE_TOKEN_STAT = "removeToken";
+ final static String TOKEN_FAILURE_STAT = "tokenFailure";
+
+ final MetricsRegistry registry;
+ final IOStatisticsStore ioStatistics;
+
+ @Metric("Rate of storage of delegation tokens and latency (milliseconds)")
+ MutableRate storeToken;
+ @Metric("Rate of update of delegation tokens and latency (milliseconds)")
+ MutableRate updateToken;
+ @Metric("Rate of removal of delegation tokens and latency (milliseconds)")
+ MutableRate removeToken;
+ @Metric("Counter of delegation tokens operation failures")
+ MutableCounterLong tokenFailure;
+
+ static DelegationTokenSecretManagerMetrics create() {
+ return DefaultMetricsSystem.instance().register(new
DelegationTokenSecretManagerMetrics());
+ }
+
+ public DelegationTokenSecretManagerMetrics() {
+ ioStatistics = iostatisticsStore()
+ .withDurationTracking(STORE_TOKEN_STAT, UPDATE_TOKEN_STAT,
REMOVE_TOKEN_STAT)
+ .withCounters(TOKEN_FAILURE_STAT)
+ .build();
+ registry = new MetricsRegistry("DelegationTokenSecretManagerMetrics");
+ LOG.debug("Initialized {}", registry);
+ }
+
+ public void addStoreToken(long value) {
Review Comment:
And probably time and the unit instead of plain "value"
##########
hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/token/delegation/TestDelegationToken.java:
##########
@@ -579,4 +615,65 @@ public void testEmptyToken() throws IOException {
assertEquals(token1, token2);
assertEquals(token1.encodeToUrlString(), token2.encodeToUrlString());
}
+
+ @Test
+ public void testDelegationTokenSecretManagerMetrics() throws Exception {
+ TestDelegationTokenSecretManager dtSecretManager =
+ new TestDelegationTokenSecretManager(24*60*60*1000,
+ 10*1000,1*1000,3600000);
Review Comment:
Extend the 3600000 too.
Issue Time Tracking
-------------------
Worklog Id: (was: 757235)
Time Spent: 2.5h (was: 2h 20m)
> Add metrics to track delegation token secret manager operations
> ---------------------------------------------------------------
>
> Key: HADOOP-18167
> URL: https://issues.apache.org/jira/browse/HADOOP-18167
> Project: Hadoop Common
> Issue Type: Improvement
> Reporter: Hector Sandoval Chaverri
> Priority: Major
> Labels: pull-request-available
> Attachments: HADOOP-18167-branch-2.10-2.patch,
> HADOOP-18167-branch-2.10.patch
>
> Time Spent: 2.5h
> Remaining Estimate: 0h
>
> New metrics to track operations that store, update and remove delegation
> tokens in implementations of AbstractDelegationTokenSecretManager. This will
> help evaluate the impact of using different secret managers and add
> optimizations.
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-issues-h...@hadoop.apache.org
