Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 30ebb792f -> f3fd5e30c
YARN-3697. FairScheduler: ContinuousSchedulingThread can fail to shutdown.
(Zhihai Xu via kasha)
(cherry picked from commit 332b520a480994b7bd56c135f7941aad30b05e9c)
Conflicts:
hadoop-yarn-project/CHANGES.txt
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3fd5e30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3fd5e30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3fd5e30
Branch: refs/heads/branch-2.7
Commit: f3fd5e30cd4d5a307315bf114b01e10c637c6d1d
Parents: 30ebb79
Author: Karthik Kambatla <[email protected]>
Authored: Sun Sep 13 18:07:43 2015 -0700
Committer: Zhihai Xu <[email protected]>
Committed: Thu Sep 17 23:35:47 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/event/TestAsyncDispatcher.java | 2 ++
.../scheduler/fair/FairScheduler.java | 7 +++++
.../scheduler/fair/TestFairScheduler.java | 31 ++++++++++++++++++++
4 files changed, 43 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fd5e30/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a40d0a4..4d9c5d7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -73,6 +73,9 @@ Release 2.7.2 - UNRELEASED
YARN-4153. TestAsyncDispatcher failed at branch-2.7 (Zhihai Xu via jianhe)
+ YARN-3697. FairScheduler: ContinuousSchedulingThread can fail to shutdown.
+ (Zhihai Xu via kasha)
+
YARN-4158. Remove duplicate close for LogWriter in
AppLogAggregatorImpl#uploadLogsForContainers (Zhihai Xu via jlowe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fd5e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
index ba0deff..018096b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/TestAsyncDispatcher.java
@@ -50,7 +50,9 @@ public class TestAsyncDispatcher {
disp.waitForEventThreadToWait();
try {
disp.getEventHandler().handle(event);
+ Assert.fail("Expected YarnRuntimeException");
} catch (YarnRuntimeException e) {
+ Assert.assertTrue(e.getCause() instanceof InterruptedException);
}
// Queue should be empty and dispatcher should not hang on close
Assert.assertTrue("Event Queue should have been empty",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fd5e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 4f427fe..5fb71ce 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1010,6 +1010,13 @@ public class FairScheduler extends
} catch (Throwable ex) {
LOG.error("Error while attempting scheduling for node " + node +
": " + ex.toString(), ex);
+ if ((ex instanceof YarnRuntimeException) &&
+ (ex.getCause() instanceof InterruptedException)) {
+ // AsyncDispatcher translates InterruptedException to
+ // YarnRuntimeException with cause InterruptedException.
+ // Need to throw InterruptedException to stop schedulingThread.
+ throw (InterruptedException)ex.getCause();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3fd5e30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 5162f46..0989a8d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -27,7 +27,10 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.isA;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.File;
@@ -3951,6 +3954,34 @@ public class TestFairScheduler extends
FairSchedulerTestBase {
}
@Test
+ public void testContinuousSchedulingInterruptedException()
+ throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ FairScheduler spyScheduler = spy(scheduler);
+ Assert.assertTrue("Continuous scheduling should be disabled.",
+ !spyScheduler.isContinuousSchedulingEnabled());
+ // Add one nodes
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ spyScheduler.handle(nodeEvent1);
+ Assert.assertEquals("We should have one alive node.",
+ 1, spyScheduler.getNumClusterNodes());
+ InterruptedException ie = new InterruptedException();
+ doThrow(new YarnRuntimeException(ie)).when(spyScheduler).
+ attemptScheduling(isA(FSSchedulerNode.class));
+ // Invoke the continuous scheduling once
+ try {
+ spyScheduler.continuousSchedulingAttempt();
+ fail("Expected InterruptedException to stop schedulingThread");
+ } catch (InterruptedException e) {
+ Assert.assertEquals(ie, e);
+ }
+ }
+
+ @Test
public void testDontAllowUndeclaredPools() throws Exception {
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
