Repository: hadoop Updated Branches: refs/heads/branch-2.6 cfbb03542 -> 4be18edf2
YARN-4380. TestResourceLocalizationService.testDownloadingResourcesOnContainerKill fails intermittently. Contributed by Varun Saxena. (cherry picked from commit 0656d2dc83af6a48a8d8d0e37cdf1f813124f366) (cherry picked from commit d76b523b020fcd76e9abdd661967ca4a931d9863) Conflicts: hadoop-yarn-project/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4be18edf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4be18edf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4be18edf Branch: refs/heads/branch-2.6 Commit: 4be18edf2f1946e7a6d55799b9db95e01c343d36 Parents: cfbb035 Author: Tsuyoshi Ozawa <oz...@apache.org> Authored: Thu Nov 26 01:10:02 2015 +0900 Committer: Junping Du <junping...@apache.org> Committed: Wed Jan 6 14:34:15 2016 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../TestResourceLocalizationService.java | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be18edf/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 684476c..fe921b5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -37,6 +37,9 @@ Release 2.6.4 - UNRELEASED YARN-4354. Public resource localization fails with NPE. (Jason Lowe via junping_du) + YARN-4380. TestResourceLocalizationService.testDownloadingResourcesOnContainerKill + fails intermittently. (Varun Saxena via ozawa) + Release 2.6.3 - 2015-12-17 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be18edf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index d19315d..f374f66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -61,6 +61,7 @@ import java.util.Set; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; @@ -1104,16 +1105,23 @@ public class TestResourceLocalizationService { private static class DummyExecutor extends DefaultContainerExecutor { private volatile boolean stopLocalization = false; + private AtomicInteger numLocalizers = new AtomicInteger(0); @Override public void startLocalizer(Path nmPrivateContainerTokensPath, InetSocketAddress nmAddr, String user, String appId, String locId, LocalDirsHandlerService dirsHandler) throws IOException, InterruptedException { + numLocalizers.incrementAndGet(); while (!stopLocalization) { Thread.yield(); } } - void setStopLocalization() { + private void waitForLocalizers(int num) { + while (numLocalizers.intValue() < num) { + Thread.yield(); + } + } + private void setStopLocalization() { stopLocalization = true; } } @@ -1256,6 +1264,10 @@ public class TestResourceLocalizationService { spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1)); dispatcher.await(); + // Wait for localizers of both container c1 and c2 to begin. + exec.waitForLocalizers(2); + LocalizerRunner locC1 = + spyService.getLocalizerRunner(c1.getContainerId().toString()); final String containerIdStr = c1.getContainerId().toString(); // Heartbeats from container localizer LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class); @@ -1323,6 +1335,10 @@ public class TestResourceLocalizationService { Set<Path> paths = Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), new Path(locPath2), new Path(locPath2 + "_tmp")); + // Wait for localizer runner thread for container c1 to finish. + while (locC1.getState() != Thread.State.TERMINATED) { + Thread.sleep(50); + } // Verify if downloading resources were submitted for deletion. verify(delService).delete(eq(user), (Path) eq(null), argThat(new DownloadingPathsMatcher(paths)));