Repository: hadoop Updated Branches: refs/heads/trunk c7d022b66 -> 6c7a9d502
YARN-3834. Scrub debug logging of tokens during resource localization. Contributed by Chris Nauroth Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c7a9d50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c7a9d50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c7a9d50 Branch: refs/heads/trunk Commit: 6c7a9d502a633b5aca75c9798f19ce4a5729014e Parents: c7d022b Author: Xuan <[email protected]> Authored: Sun Jun 21 17:13:44 2015 -0700 Committer: Xuan <[email protected]> Committed: Sun Jun 21 17:13:44 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../localizer/ResourceLocalizationService.java | 29 +++++++++++++++++++- .../TestResourceLocalizationService.java | 10 +++++-- 3 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c7a9d50/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d89c285..b50f490 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -312,6 +312,9 @@ Release 2.8.0 - UNRELEASED YARN-3148. Allow CORS related headers to passthrough in WebAppProxyServlet. (Varun Saxena via devaraj) + YARN-3834. Scrub debug logging of tokens during resource localization. + (Chris Nauroth via xgong) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c7a9d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 54c31c2..d6e0903 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -51,6 +51,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; +import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -1208,7 +1209,7 @@ public class ResourceLocalizationService extends CompositeService if (LOG.isDebugEnabled()) { for (Token<? extends TokenIdentifier> tk : credentials .getAllTokens()) { - LOG.debug(tk.getService() + " : " + tk.encodeToUrlString()); + LOG.debug(tk + " : " + buildTokenFingerprint(tk)); } } if (UserGroupInformation.isSecurityEnabled()) { @@ -1228,6 +1229,32 @@ public class ResourceLocalizationService extends CompositeService } + /** + * Returns a fingerprint of a token. The fingerprint is suitable for use in + * logging, because it cannot be used to determine the secret. The + * fingerprint is built using the first 10 bytes of a SHA-256 hash of the + * string encoding of the token. The returned string contains the hex + * representation of each byte, delimited by a space. + * + * @param tk token + * @return token fingerprint + * @throws IOException if there is an I/O error + */ + @VisibleForTesting + static String buildTokenFingerprint(Token<? extends TokenIdentifier> tk) + throws IOException { + char[] digest = DigestUtils.sha256Hex(tk.encodeToUrlString()).toCharArray(); + StringBuilder fingerprint = new StringBuilder(); + for (int i = 0; i < 10; ++i) { + if (i > 0) { + fingerprint.append(' '); + } + fingerprint.append(digest[2 * i]); + fingerprint.append(digest[2 * i + 1]); + } + return fingerprint.toString(); + } + static class CacheCleanup extends Thread { private final Dispatcher dispatcher; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c7a9d50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index a02b2b0..c515506 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -2035,7 +2035,7 @@ public class TestResourceLocalizationService { } private static Container getMockContainer(ApplicationId appId, int id, - String user) { + String user) throws IOException { Container c = mock(Container.class); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1); @@ -2043,7 +2043,13 @@ public class TestResourceLocalizationService { when(c.getUser()).thenReturn(user); when(c.getContainerId()).thenReturn(cId); Credentials creds = new Credentials(); - creds.addToken(new Text("tok" + id), getToken(id)); + Token<? extends TokenIdentifier> tk = getToken(id); + String fingerprint = ResourceLocalizationService.buildTokenFingerprint(tk); + assertNotNull(fingerprint); + assertTrue( + "Expected token fingerprint of 10 hex bytes delimited by space.", + fingerprint.matches("^(([0-9a-f]){2} ){9}([0-9a-f]){2}$")); + creds.addToken(new Text("tok" + id), tk); when(c.getCredentials()).thenReturn(creds); when(c.toString()).thenReturn(cId.toString()); return c;
