Repository: tez Updated Branches: refs/heads/branch-0.8 bf3666b14 -> ef5904f60
TEZ-3600. Fix flaky test: TestTokenCache. Contributed by Harish Jaiprakash (cherry picked from commit a77d22dd8ce281ffc24b42b634abb972cb552c78) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/ef5904f6 Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/ef5904f6 Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/ef5904f6 Branch: refs/heads/branch-0.8 Commit: ef5904f60fd656fc95bb691c3493804bf1d7330d Parents: bf3666b Author: Siddharth Seth <[email protected]> Authored: Fri Feb 3 13:46:33 2017 -0800 Committer: Harish JP <[email protected]> Committed: Wed Apr 19 21:32:35 2017 +0530 ---------------------------------------------------------------------- .../main/java/org/apache/tez/common/security/TokenCache.java | 8 +++++--- .../java/org/apache/tez/common/security/TestTokenCache.java | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/ef5904f6/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java index 0ce5844..fc2c07d 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java @@ -80,13 +80,15 @@ public class TokenCache { static void obtainTokensForFileSystemsInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { Set<FileSystem> fsSet = new HashSet<FileSystem>(); + boolean limitExceeded = false; for(Path p: ps) { FileSystem fs = p.getFileSystem(conf); - if (fsSet.size() == MAX_FS_OBJECTS) { + if (!limitExceeded && fsSet.size() == MAX_FS_OBJECTS) { LOG.warn("No of FileSystem objects exceeds {}, updating tokens for all paths. This can" + - " happen when fs.<scheme>.impl.disable.cache is set to true."); + " happen when fs.<scheme>.impl.disable.cache is set to true.", MAX_FS_OBJECTS); + limitExceeded = true; } - if (fsSet.size() >= MAX_FS_OBJECTS) { + if (limitExceeded) { // Too many fs objects are being created, most likely the cache is disabled. Prevent an // OOM and just directly invoke instead of adding to the set. obtainTokensForFileSystemsInternal(fs, credentials, conf); http://git-wip-us.apache.org/repos/asf/tez/blob/ef5904f6/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java index 6fc6daa..59488b6 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -54,6 +55,8 @@ public class TestTokenCache { public static void setup() throws Exception { conf = new Configuration(); conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); + conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, + InetSocketAddress.createUnresolved("127.0.0.1", 8032)); renewer = Master.getMasterPrincipal(conf); }
