Author: jlowe
Date: Wed Mar 5 01:01:08 2014
New Revision: 1574272
URL: http://svn.apache.org/r1574272
Log:
svn merge -c 1555161 FIXES: MAPREDUCE-5689. MRAppMaster does not preempt
reducers when scheduled maps cannot be fulfilled. Contributed by Lohit
Vijayarenu
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1574272&r1=1574271&r2=1574272&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed
Mar 5 01:01:08 2014
@@ -28,6 +28,9 @@ Release 0.23.11 - UNRELEASED
RMContainerAllocator$AssignedRequests.preemptReduce() violates the
comparator contract (Gera Shegalov via kasha)
+ MAPREDUCE-5689. MRAppMaster does not preempt reducers when scheduled maps
+ cannot be fulfilled. (lohit via kasha)
+
Release 0.23.10 - 2013-12-09
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1574272&r1=1574271&r2=1574272&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Wed Mar 5 01:01:08 2014
@@ -213,7 +213,8 @@ public class RMContainerAllocator extend
int completedMaps = getJob().getCompletedMaps();
int completedTasks = completedMaps + getJob().getCompletedReduces();
- if (lastCompletedTasks != completedTasks) {
+ if ((lastCompletedTasks != completedTasks) ||
+ (scheduledRequests.maps.size() > 0)) {
lastCompletedTasks = completedTasks;
recalculateReduceSchedule = true;
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java?rev=1574272&r1=1574271&r2=1574272&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
Wed Mar 5 01:01:08 2014
@@ -1428,6 +1428,21 @@ public class TestRMContainerAllocator {
numPendingReduces,
maxReduceRampupLimit, reduceSlowStart);
verify(allocator).rampDownReduces(anyInt());
+
+ // Test reduce ramp-down for when there are scheduled maps
+ // Since we have two scheduled Maps, rampDownReducers
+ // should be invoked twice.
+ scheduledMaps = 2;
+ assignedReduces = 2;
+ doReturn(10 * 1024).when(allocator).getMemLimit();
+ allocator.scheduleReduces(
+ totalMaps, succeededMaps,
+ scheduledMaps, scheduledReduces,
+ assignedMaps, assignedReduces,
+ mapResourceReqt, reduceResourceReqt,
+ numPendingReduces,
+ maxReduceRampupLimit, reduceSlowStart);
+ verify(allocator, times(2)).rampDownReduces(anyInt());
}
private static class RecalculateContainerAllocator extends
MyContainerAllocator {
