Repository: hadoop Updated Branches: refs/heads/YARN-1051 a6e38a538 -> c47464aba
YARN-2611. Fixing jenkins findbugs warning and TestRMWebServicesCapacitySched for branch YARN-1051. Contributed by Subru Krishnan and Carlo Curino. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c47464ab Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c47464ab Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c47464ab Branch: refs/heads/YARN-1051 Commit: c47464aba407d1dafe10be23fe454f0489cc4367 Parents: a6e38a5 Author: subru <[email protected]> Authored: Fri Sep 26 10:48:12 2014 -0700 Committer: subru <[email protected]> Committed: Fri Sep 26 10:48:12 2014 -0700 ---------------------------------------------------------------------- YARN-1051-CHANGES.txt | 3 ++ .../reservation/CapacityOverTimePolicy.java | 7 ++++- .../CapacitySchedulerPlanFollower.java | 2 +- .../reservation/InMemoryPlan.java | 6 ++-- .../reservation/ReservationInterval.java | 30 ++++++++++++++++++++ .../reservation/SimpleCapacityReplanner.java | 5 ++++ .../webapp/TestRMWebServicesCapacitySched.java | 4 +-- 7 files changed, 50 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/YARN-1051-CHANGES.txt ---------------------------------------------------------------------- diff --git a/YARN-1051-CHANGES.txt b/YARN-1051-CHANGES.txt index 7620cc3..5cd1136 100644 --- a/YARN-1051-CHANGES.txt +++ b/YARN-1051-CHANGES.txt @@ -29,3 +29,6 @@ delegate. (Subru Krishnan and Carlo Curino via subru) YARN-2576. Fixing compilation, javadocs and audit issues to pass test patch in branch. (Subru Krishnan and Carlo Curino via subru) + +YARN-2611. Fixing jenkins findbugs warning and TestRMWebServicesCapacitySched +for branch YARN-1051. (Subru Krishnan and Carlo Curino via subru) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java index 680f9ae..7552e8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacityOverTimePolicy.java @@ -62,6 +62,11 @@ public class CapacityOverTimePolicy implements SharingPolicy { // it should be easy to remove this limitation @Override public void init(String reservationQueuePath, Configuration conf) { + if (!(conf instanceof CapacitySchedulerConfiguration)) { + throw new IllegalArgumentException("Unexpected conf type: " + + conf.getClass().getSimpleName() + " only supported conf is: " + + CapacitySchedulerConfiguration.class.getSimpleName()); + } this.conf = (CapacitySchedulerConfiguration) conf; validWindow = this.conf.getReservationWindow(reservationQueuePath); maxInst = this.conf.getInstantaneousMaxCapacity(reservationQueuePath) / 100; @@ -203,7 +208,7 @@ public class CapacityOverTimePolicy implements SharingPolicy { * The comparison/multiplication behaviors of IntegralResource are consistent * with the DefaultResourceCalculator. */ - public class IntegralResource { + private static class IntegralResource { long memory; long vcores; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java index cfa172c..0c0fbc0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/CapacitySchedulerPlanFollower.java @@ -326,7 +326,7 @@ public class CapacitySchedulerPlanFollower implements PlanFollower { return currentReservations; } - private class ReservationAllocationComparator implements + private static class ReservationAllocationComparator implements Comparator<ReservationAllocation> { CapacityScheduler scheduler; long now; http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java index 8394a49..ce2e7d7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/InMemoryPlan.java @@ -138,7 +138,7 @@ class InMemoryPlan implements Plan { rleSparseVector.removeInterval(r.getKey(), r.getValue()); } if (resAlloc.isEmpty()) { - userResourceAlloc.remove(resAlloc); + userResourceAlloc.remove(user); } } @@ -311,9 +311,9 @@ class InMemoryPlan implements Plan { public void archiveCompletedReservations(long tick) { // Since we are looking for old reservations, read lock is optimal LOG.debug("Running archival at time: {}", tick); - readLock.lock(); List<InMemoryReservationAllocation> expiredReservations = new ArrayList<InMemoryReservationAllocation>(); + readLock.lock(); // archive reservations and delete the ones which are beyond // the reservation policy "window" try { @@ -351,9 +351,9 @@ class InMemoryPlan implements Plan { @Override public Set<ReservationAllocation> getReservationsAtTime(long tick) { - readLock.lock(); ReservationInterval searchInterval = new ReservationInterval(tick, Long.MAX_VALUE); + readLock.lock(); try { SortedMap<ReservationInterval, Set<InMemoryReservationAllocation>> reservations = currentReservations.headMap(searchInterval, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java index aabbcb3..1ca2969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInterval.java @@ -78,6 +78,36 @@ public class ReservationInterval implements Comparable<ReservationInterval> { } } + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (endTime ^ (endTime >>> 32)); + result = prime * result + (int) (startTime ^ (startTime >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof ReservationInterval)) { + return false; + } + ReservationInterval other = (ReservationInterval) obj; + if (endTime != other.endTime) { + return false; + } + if (startTime != other.startTime) { + return false; + } + return true; + } + public String toString() { return "[" + startTime + ", " + endTime + "]"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java index 5378405..e38dd3cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/SimpleCapacityReplanner.java @@ -70,6 +70,11 @@ public class SimpleCapacityReplanner implements Planner { @Override public void init(String planQueueName, Configuration conf) { + if (!(conf instanceof CapacitySchedulerConfiguration)) { + throw new IllegalArgumentException("Unexpected conf type: " + + conf.getClass().getSimpleName() + " only supported conf is: " + + CapacitySchedulerConfiguration.class.getSimpleName()); + } this.lengthOfCheckZone = ((CapacitySchedulerConfiguration) conf) .getEnforcementWindow(planQueueName); http://git-wip-us.apache.org/repos/asf/hadoop/blob/c47464ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index e57e5cf..e58c30f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -357,10 +357,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTest { private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 11; + int numExpectedElements = 12; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 21; + numExpectedElements = 22; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length());
