Repository: tez Updated Branches: refs/heads/master 403a85d76 -> a77d22dd8
TEZ-3600. Fix flaky test: TestTokenCache. Contributed by Harish Jaiprakash Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/a77d22dd Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/a77d22dd Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/a77d22dd Branch: refs/heads/master Commit: a77d22dd8ce281ffc24b42b634abb972cb552c78 Parents: 403a85d Author: Siddharth Seth <[email protected]> Authored: Fri Feb 3 13:46:33 2017 -0800 Committer: Siddharth Seth <[email protected]> Committed: Fri Feb 3 13:46:33 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../main/java/org/apache/tez/common/security/TokenCache.java | 8 +++++--- .../java/org/apache/tez/common/security/TestTokenCache.java | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/a77d22dd/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 5d04ecb..bf9c9bc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-3600. Fix flaky test: TestTokenCache TEZ-3589. add a unit test for amKeepAlive not being shutdown if an app takes a long time to launch. TEZ-3417. Reduce sleep time on AM shutdown to reduce test runtimes TEZ-3582. Exception swallowed in PipelinedSorter causing incorrect results. http://git-wip-us.apache.org/repos/asf/tez/blob/a77d22dd/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java index 0ce5844..fc2c07d 100644 --- a/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java +++ b/tez-api/src/main/java/org/apache/tez/common/security/TokenCache.java @@ -80,13 +80,15 @@ public class TokenCache { static void obtainTokensForFileSystemsInternal(Credentials credentials, Path[] ps, Configuration conf) throws IOException { Set<FileSystem> fsSet = new HashSet<FileSystem>(); + boolean limitExceeded = false; for(Path p: ps) { FileSystem fs = p.getFileSystem(conf); - if (fsSet.size() == MAX_FS_OBJECTS) { + if (!limitExceeded && fsSet.size() == MAX_FS_OBJECTS) { LOG.warn("No of FileSystem objects exceeds {}, updating tokens for all paths. This can" + - " happen when fs.<scheme>.impl.disable.cache is set to true."); + " happen when fs.<scheme>.impl.disable.cache is set to true.", MAX_FS_OBJECTS); + limitExceeded = true; } - if (fsSet.size() >= MAX_FS_OBJECTS) { + if (limitExceeded) { // Too many fs objects are being created, most likely the cache is disabled. Prevent an // OOM and just directly invoke instead of adding to the set. obtainTokensForFileSystemsInternal(fs, credentials, conf); http://git-wip-us.apache.org/repos/asf/tez/blob/a77d22dd/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java index 6fc6daa..59488b6 100644 --- a/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java +++ b/tez-api/src/test/java/org/apache/tez/common/security/TestTokenCache.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; import java.net.URISyntaxException; @@ -54,6 +55,8 @@ public class TestTokenCache { public static void setup() throws Exception { conf = new Configuration(); conf.set(YarnConfiguration.RM_PRINCIPAL, "mapred/host@REALM"); + conf.setSocketAddr(YarnConfiguration.RM_ADDRESS, + InetSocketAddress.createUnresolved("127.0.0.1", 8032)); renewer = Master.getMasterPrincipal(conf); }
