This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 57e88a6 YARN-9552. FairScheduler: NODE_UPDATE can cause
NoSuchElementException. Contributed by Peter Bacsko.
57e88a6 is described below
commit 57e88a63cfb6d50507c92f268b36f3b23debe7e5
Author: Szilard Nemeth <snem...@apache.org>
AuthorDate: Wed Oct 9 14:19:56 2019 +0200
YARN-9552. FairScheduler: NODE_UPDATE can cause NoSuchElementException.
Contributed by Peter Bacsko.
---
.../scheduler/AppSchedulingInfo.java | 11 +++---
.../scheduler/fair/FSAppAttempt.java | 4 +--
.../scheduler/fair/TestFSAppAttempt.java | 42 ++++++++++++++++++++++
3 files changed, 51 insertions(+), 6 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 8074f06..21f052a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -477,14 +477,17 @@ public class AppSchedulingInfo {
}
public PendingAsk getNextPendingAsk() {
+ readLock.lock();
try {
- readLock.lock();
- SchedulerRequestKey firstRequestKey = schedulerKeys.first();
- return getPendingAsk(firstRequestKey, ResourceRequest.ANY);
+ if (!schedulerKeys.isEmpty()) {
+ SchedulerRequestKey firstRequestKey = schedulerKeys.first();
+ return getPendingAsk(firstRequestKey, ResourceRequest.ANY);
+ } else {
+ return null;
+ }
} finally {
readLock.unlock();
}
-
}
public PendingAsk getPendingAsk(SchedulerRequestKey schedulerKey) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 2355fb7..0eedec0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -936,8 +936,8 @@ public class FSAppAttempt extends
SchedulerApplicationAttempt
if (!isAmRunning() && !getUnmanagedAM()) {
// Return true if we have not ask, or queue is not be able to run app's
AM
PendingAsk ask = appSchedulingInfo.getNextPendingAsk();
- if (ask.getCount() == 0 || !getQueue().canRunAppAM(
- ask.getPerAllocationResource())) {
+ if (ask != null && (ask.getCount() == 0 || !getQueue().canRunAppAM(
+ ask.getPerAllocationResource()))) {
return true;
}
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
index 51ffd23..ad00a1b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
@@ -19,7 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -33,8 +36,12 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
@@ -341,6 +348,41 @@ public class TestFSAppAttempt extends
FairSchedulerTestBase {
assertEquals(clusterResource, spyApp.getHeadroom());
}
+ /**
+ * Ensure that no pending ask request inside appSchedulingInfo
+ * does not result in an error.
+ */
+ @Test
+ public void testNoNextPendingAsk() {
+ FSLeafQueue queue = Mockito.mock(FSLeafQueue.class);
+ ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ ConcurrentMap<ApplicationId, RMApp> rmApps = new ConcurrentHashMap<>();
+ RMApp rmApp = Mockito.mock(RMApp.class);
+ rmApps.put(applicationAttemptId.getApplicationId(), rmApp);
+ ApplicationSubmissionContext appContext =
+ Mockito.mock(ApplicationSubmissionContext.class);
+ Mockito.when(appContext.getUnmanagedAM()).thenReturn(false);
+ LogAggregationContext logAggregationContext =
+ Mockito.mock(LogAggregationContext.class);
+ Mockito.when(appContext.getLogAggregationContext())
+ .thenReturn(logAggregationContext);
+ Mockito.when(rmApp.getApplicationSchedulingEnvs())
+ .thenReturn(new HashMap<>());
+ Mockito.when(rmApp.getApplicationSubmissionContext())
+ .thenReturn(appContext);
+ Mockito.when(rmContext.getRMApps()).thenReturn(rmApps);
+ FSAppAttempt schedulerApp =
+ new FSAppAttempt(scheduler, applicationAttemptId, "user1", queue,
+ null, rmContext);
+ schedulerApp.setAmRunning(false);
+ FSSchedulerNode schedulerNode = Mockito.mock(FSSchedulerNode.class);
+
+ Resource resource = schedulerApp.assignContainer(schedulerNode);
+
+ assertEquals(Resources.none(), resource);
+ }
+
private static long min(long value1, long value2, long value3) {
return Math.min(Math.min(value1, value2), value3);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
