Author: bobby Date: Tue May 8 15:26:11 2012 New Revision: 1335587 URL: http://svn.apache.org/viewvc?rev=1335587&view=rev Log: svn merge -c 1335585 FIXES: MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Sharp via bobby)
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1335587&r1=1335586&r2=1335587&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue May 8 15:26:11 2012 @@ -216,6 +216,9 @@ Release 0.23.3 - UNRELEASED OPTIMIZATIONS + MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn + Sharp via bobby) + BUG FIXES MAPREDUCE-4092. commitJob Exception does not fail job (Jon Eagles via Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java?rev=1335587&r1=1335586&r2=1335587&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/TokenCache.java Tue May 8 15:26:11 2012 @@ -19,7 +19,9 @@ package org.apache.hadoop.mapreduce.security; import java.io.IOException; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -92,8 +94,11 @@ public class TokenCache { static void obtainTokensForNamenodesInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { + Set<FileSystem> fsSet = new HashSet<FileSystem>(); for(Path p: ps) { - FileSystem fs = FileSystem.get(p.toUri(), conf); + fsSet.add(p.getFileSystem(conf)); + } + for (FileSystem fs : fsSet) { obtainTokensForNamenodesInternal(fs, credentials, conf); } } Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1335587&r1=1335586&r2=1335587&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/security/TestTokenCache.java Tue May 8 15:26:11 2012 @@ -252,6 +252,26 @@ public class TestTokenCache { } @Test + public void testSingleTokenFetch() throws Exception { + Configuration conf = new Configuration(); + conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); + String renewer = Master.getMasterPrincipal(conf); + Credentials credentials = new Credentials(); + + FileSystem mockFs = mock(FileSystem.class); + when(mockFs.getCanonicalServiceName()).thenReturn("host:0"); + when(mockFs.getUri()).thenReturn(new URI("mockfs://host:0")); + + Path mockPath = mock(Path.class); + when(mockPath.getFileSystem(conf)).thenReturn(mockFs); + + Path[] paths = new Path[]{ mockPath, mockPath }; + when(mockFs.getDelegationTokens("me", credentials)).thenReturn(null); + TokenCache.obtainTokensForNamenodesInternal(credentials, paths, conf); + verify(mockFs, times(1)).getDelegationTokens(renewer, credentials); + } + + @Test public void testCleanUpTokenReferral() throws Exception { Configuration conf = new Configuration(); conf.set(MRJobConfig.MAPREDUCE_JOB_CREDENTIALS_BINARY, "foo");