http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java deleted file mode 100644 index de94dcd..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java +++ /dev/null @@ -1,604 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -package org.apache.hadoop.yarn.server.resourcemanager.reservation; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.hadoop.yarn.api.records.ReservationDefinition; -import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.ReservationRequest; -import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; -import org.apache.hadoop.yarn.api.records.ReservationRequests; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.apache.hadoop.yarn.util.resource.ResourceCalculator; -import org.apache.hadoop.yarn.util.resource.Resources; -import org.junit.Before; -import org.junit.Test; -import org.mortbay.log.Log; - -public class TestGreedyReservationAgent { - - ReservationAgent agent; - InMemoryPlan plan; - Resource minAlloc = Resource.newInstance(1024, 1); - ResourceCalculator res = new DefaultResourceCalculator(); - Resource maxAlloc = Resource.newInstance(1024 * 8, 8); - Random rand = new Random(); - long step; - - @Before - public void setup() throws Exception { - - long seed = rand.nextLong(); - rand.setSeed(seed); - Log.info("Running with seed: " + seed); - - // setting completely loose quotas - long timeWindow = 1000000L; - Resource clusterCapacity = Resource.newInstance(100 * 1024, 100); - step = 1000L; - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); - String reservationQ = testUtil.getFullReservationQueueName(); - - float instConstraint = 100; - float avgConstraint = 100; - - ReservationSchedulerConfiguration conf = - ReservationSystemTestUtil.createConf(reservationQ, timeWindow, - instConstraint, avgConstraint); - CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); - policy.init(reservationQ, conf); - agent = new GreedyReservationAgent(); - - QueueMetrics queueMetrics = mock(QueueMetrics.class); - - plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, - res, minAlloc, maxAlloc, "dedicated", null, true); - } - - @SuppressWarnings("javadoc") - @Test - public void testSimple() throws PlanningException { - - prepareBasicPlan(); - - // create a request with a single atomic ask - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(5 * step); - rr.setDeadline(20 * step); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 10, 5, 10 * step); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setReservationResources(Collections.singletonList(r)); - rr.setReservationRequests(reqs); - - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - agent.createReservation(reservationID, "u1", plan, rr); - - assertTrue("Agent-based allocation failed", reservationID != null); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 3); - - ReservationAllocation cs = plan.getReservationById(reservationID); - - System.out.println("--------AFTER SIMPLE ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - for (long i = 10 * step; i < 20 * step; i++) { - assertTrue( - "Agent-based allocation unexpected", - Resources.equals(cs.getResourcesAtTime(i), - Resource.newInstance(2048 * 10, 2 * 10))); - } - - } - - @Test - public void testOrder() throws PlanningException { - prepareBasicPlan(); - - // create a completely utilized segment around time 30 - int[] f = { 100, 100 }; - - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation( - ReservationSystemTestUtil.getNewReservationId(), null, "u1", - "dedicated", 30 * step, 30 * step + f.length * step, - ReservationSystemTestUtil.generateAllocation(30 * step, step, f), - res, minAlloc))); - - // create a chain of 4 RR, mixing gang and non-gang - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(0 * step); - rr.setDeadline(70 * step); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 10, 1, 10 * step); - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 10, 10, 20 * step); - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - list.add(r2); - list.add(r); - list.add(r2); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - // submit to agent - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - agent.createReservation(reservationID, "u1", plan, rr); - - // validate - assertTrue("Agent-based allocation failed", reservationID != null); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 4); - - ReservationAllocation cs = plan.getReservationById(reservationID); - - assertTrue(cs.toString(), check(cs, 0 * step, 10 * step, 20, 1024, 1)); - assertTrue(cs.toString(), check(cs, 10 * step, 30 * step, 10, 1024, 1)); - assertTrue(cs.toString(), check(cs, 40 * step, 50 * step, 20, 1024, 1)); - assertTrue(cs.toString(), check(cs, 50 * step, 70 * step, 10, 1024, 1)); - - System.out.println("--------AFTER ORDER ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - @Test - public void testOrderNoGapImpossible() throws PlanningException { - prepareBasicPlan(); - // create a completely utilized segment at time 30 - int[] f = { 100, 100 }; - - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation( - ReservationSystemTestUtil.getNewReservationId(), null, "u1", - "dedicated", 30 * step, 30 * step + f.length * step, - ReservationSystemTestUtil.generateAllocation(30 * step, step, f), - res, minAlloc))); - - // create a chain of 4 RR, mixing gang and non-gang - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(0L); - - rr.setDeadline(70L); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER_NO_GAP); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 10, 1, 10); - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 10, 10, 20); - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - list.add(r2); - list.add(r); - list.add(r2); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - boolean result = false; - try { - // submit to agent - result = agent.createReservation(reservationID, "u1", plan, rr); - fail(); - } catch (PlanningException p) { - // expected - } - - // validate - assertFalse("Agent-based allocation should have failed", result); - assertTrue("Agent-based allocation should have failed", plan - .getAllReservations().size() == 3); - - System.out - .println("--------AFTER ORDER_NO_GAP IMPOSSIBLE ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - @Test - public void testOrderNoGap() throws PlanningException { - prepareBasicPlan(); - // create a chain of 4 RR, mixing gang and non-gang - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(0 * step); - rr.setDeadline(60 * step); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ORDER_NO_GAP); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 10, 1, 10 * step); - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 10, 10, 20 * step); - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - list.add(r2); - list.add(r); - list.add(r2); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - rr.setReservationRequests(reqs); - - // submit to agent - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - agent.createReservation(reservationID, "u1", plan, rr); - - System.out.println("--------AFTER ORDER ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - // validate - assertTrue("Agent-based allocation failed", reservationID != null); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 3); - - ReservationAllocation cs = plan.getReservationById(reservationID); - - assertTrue(cs.toString(), check(cs, 0 * step, 10 * step, 20, 1024, 1)); - assertTrue(cs.toString(), check(cs, 10 * step, 30 * step, 10, 1024, 1)); - assertTrue(cs.toString(), check(cs, 30 * step, 40 * step, 20, 1024, 1)); - assertTrue(cs.toString(), check(cs, 40 * step, 60 * step, 10, 1024, 1)); - - } - - @Test - public void testSingleSliding() throws PlanningException { - prepareBasicPlan(); - - // create a single request for which we need subsequent (tight) packing. - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(100 * step); - rr.setDeadline(120 * step); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 200, 10, 10 * step); - - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - // submit to agent - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - agent.createReservation(reservationID, "u1", plan, rr); - - // validate results, we expect the second one to be accepted - assertTrue("Agent-based allocation failed", reservationID != null); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 3); - - ReservationAllocation cs = plan.getReservationById(reservationID); - - assertTrue(cs.toString(), check(cs, 100 * step, 120 * step, 100, 1024, 1)); - - System.out.println("--------AFTER packed ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - @Test - public void testAny() throws PlanningException { - prepareBasicPlan(); - // create an ANY request, with an impossible step (last in list, first - // considered), - // and two satisfiable ones. We expect the second one to be returned. - - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(100 * step); - rr.setDeadline(120 * step); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ANY); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 5, 5, 10 * step); - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 10, 5, 10 * step); - ReservationRequest r3 = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 110, 110, 10 * step); - - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - list.add(r2); - list.add(r3); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - // submit to agent - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - boolean res = agent.createReservation(reservationID, "u1", plan, rr); - - // validate results, we expect the second one to be accepted - assertTrue("Agent-based allocation failed", res); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 3); - - ReservationAllocation cs = plan.getReservationById(reservationID); - - assertTrue(cs.toString(), check(cs, 110 * step, 120 * step, 20, 1024, 1)); - - System.out.println("--------AFTER ANY ALLOCATION (queue: " + reservationID - + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - @Test - public void testAnyImpossible() throws PlanningException { - prepareBasicPlan(); - // create an ANY request, with all impossible alternatives - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(100L); - rr.setDeadline(120L); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ANY); - - // longer than arrival-deadline - ReservationRequest r1 = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 35, 5, 30); - // above max cluster size - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 110, 110, 10); - - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r1); - list.add(r2); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - boolean result = false; - try { - // submit to agent - result = agent.createReservation(reservationID, "u1", plan, rr); - fail(); - } catch (PlanningException p) { - // expected - } - // validate results, we expect the second one to be accepted - assertFalse("Agent-based allocation should have failed", result); - assertTrue("Agent-based allocation should have failed", plan - .getAllReservations().size() == 2); - - System.out.println("--------AFTER ANY IMPOSSIBLE ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - @Test - public void testAll() throws PlanningException { - prepareBasicPlan(); - // create an ALL request - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(100 * step); - rr.setDeadline(120 * step); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 5, 5, 10 * step); - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 10, 10, 20 * step); - - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - list.add(r2); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - // submit to agent - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - agent.createReservation(reservationID, "u1", plan, rr); - - // validate results, we expect the second one to be accepted - assertTrue("Agent-based allocation failed", reservationID != null); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 3); - - ReservationAllocation cs = plan.getReservationById(reservationID); - - assertTrue(cs.toString(), check(cs, 100 * step, 110 * step, 20, 1024, 1)); - assertTrue(cs.toString(), check(cs, 110 * step, 120 * step, 25, 1024, 1)); - - System.out.println("--------AFTER ALL ALLOCATION (queue: " + reservationID - + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - @Test - public void testAllImpossible() throws PlanningException { - prepareBasicPlan(); - // create an ALL request, with an impossible combination, it should be - // rejected, and allocation remain unchanged - ReservationDefinition rr = new ReservationDefinitionPBImpl(); - rr.setArrival(100L); - rr.setDeadline(120L); - ReservationRequests reqs = new ReservationRequestsPBImpl(); - reqs.setInterpreter(ReservationRequestInterpreter.R_ALL); - ReservationRequest r = ReservationRequest.newInstance( - Resource.newInstance(1024, 1), 55, 5, 10); - ReservationRequest r2 = ReservationRequest.newInstance( - Resource.newInstance(2048, 2), 55, 5, 20); - - List<ReservationRequest> list = new ArrayList<ReservationRequest>(); - list.add(r); - list.add(r2); - reqs.setReservationResources(list); - rr.setReservationRequests(reqs); - - ReservationId reservationID = ReservationSystemTestUtil - .getNewReservationId(); - boolean result = false; - try { - // submit to agent - result = agent.createReservation(reservationID, "u1", plan, rr); - fail(); - } catch (PlanningException p) { - // expected - } - - // validate results, we expect the second one to be accepted - assertFalse("Agent-based allocation failed", result); - assertTrue("Agent-based allocation failed", plan.getAllReservations() - .size() == 2); - - System.out.println("--------AFTER ALL IMPOSSIBLE ALLOCATION (queue: " - + reservationID + ")----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - - } - - private void prepareBasicPlan() throws PlanningException { - - // insert in the reservation a couple of controlled reservations, to create - // conditions for assignment that are non-empty - - int[] f = { 10, 10, 20, 20, 20, 10, 10 }; - - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation( - ReservationSystemTestUtil.getNewReservationId(), null, "u1", - "dedicated", 0L, 0L + f.length * step, ReservationSystemTestUtil - .generateAllocation(0, step, f), res, minAlloc))); - - int[] f2 = { 5, 5, 5, 5, 5, 5, 5 }; - Map<ReservationInterval, Resource> alloc = - ReservationSystemTestUtil.generateAllocation(5000, step, f2); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation( - ReservationSystemTestUtil.getNewReservationId(), null, "u1", - "dedicated", 5000, 5000 + f2.length * step, alloc, res, minAlloc))); - - System.out.println("--------BEFORE AGENT----------"); - System.out.println(plan.toString()); - System.out.println(plan.toCumulativeString()); - } - - private boolean check(ReservationAllocation cs, long start, long end, - int containers, int mem, int cores) { - - boolean res = true; - for (long i = start; i < end; i++) { - res = res - && Resources.equals(cs.getResourcesAtTime(i), - Resource.newInstance(mem * containers, cores * containers)); - } - return res; - } - - public void testStress(int numJobs) throws PlanningException, IOException { - - long timeWindow = 1000000L; - Resource clusterCapacity = Resource.newInstance(500 * 100 * 1024, 500 * 32); - step = 1000L; - ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); - CapacityScheduler scheduler = testUtil.mockCapacityScheduler(500 * 100); - String reservationQ = testUtil.getFullReservationQueueName(); - float instConstraint = 100; - float avgConstraint = 100; - ReservationSchedulerConfiguration conf = - ReservationSystemTestUtil.createConf(reservationQ, timeWindow, - instConstraint, avgConstraint); - CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); - policy.init(reservationQ, conf); - - plan = new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, agent, - clusterCapacity, step, res, minAlloc, maxAlloc, "dedicated", null, true); - - int acc = 0; - List<ReservationDefinition> list = new ArrayList<ReservationDefinition>(); - for (long i = 0; i < numJobs; i++) { - list.add(ReservationSystemTestUtil.generateRandomRR(rand, i)); - } - - long start = System.currentTimeMillis(); - for (int i = 0; i < numJobs; i++) { - - try { - if (agent.createReservation( - ReservationSystemTestUtil.getNewReservationId(), "u" + i % 100, - plan, list.get(i))) { - acc++; - } - } catch (PlanningException p) { - // ignore exceptions - } - } - - long end = System.currentTimeMillis(); - System.out.println("Submitted " + numJobs + " jobs " + " accepted " + acc - + " in " + (end - start) + "ms"); - } - - public static void main(String[] arg) { - - // run a stress test with by default 1000 random jobs - int numJobs = 1000; - if (arg.length > 0) { - numJobs = Integer.parseInt(arg[0]); - } - - try { - TestGreedyReservationAgent test = new TestGreedyReservationAgent(); - test.setup(); - test.testStress(numJobs); - } catch (Exception e) { - e.printStackTrace(); - } - } - -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java index 722fb29..b6d24b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestInMemoryPlan.java @@ -34,6 +34,8 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.Planner; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java index 1e15618..809892c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java index d0f4dc6..f0cc49c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestRLESparseResourceAllocation.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -164,6 +164,53 @@ public class TestRLESparseResourceAllocation { Assert.assertTrue(rleSparseVector.isEmpty()); } + @Test + public void testToIntervalMap() { + ResourceCalculator resCalc = new DefaultResourceCalculator(); + Resource minAlloc = Resource.newInstance(1, 1); + RLESparseResourceAllocation rleSparseVector = + new RLESparseResourceAllocation(resCalc, minAlloc); + Map<ReservationInterval, Resource> mapAllocations; + + // Check empty + mapAllocations = rleSparseVector.toIntervalMap(); + Assert.assertTrue(mapAllocations.isEmpty()); + + // Check full + int[] alloc = { 0, 5, 10, 10, 5, 0, 5, 0 }; + int start = 100; + Set<Entry<ReservationInterval, Resource>> inputs = + generateAllocation(start, alloc, false).entrySet(); + for (Entry<ReservationInterval, Resource> ip : inputs) { + rleSparseVector.addInterval(ip.getKey(), ip.getValue()); + } + mapAllocations = rleSparseVector.toIntervalMap(); + Assert.assertTrue(mapAllocations.size() == 5); + for (Entry<ReservationInterval, Resource> entry : mapAllocations + .entrySet()) { + ReservationInterval interval = entry.getKey(); + Resource resource = entry.getValue(); + if (interval.getStartTime() == 101L) { + Assert.assertTrue(interval.getEndTime() == 102L); + Assert.assertEquals(resource, Resource.newInstance(5 * 1024, 5)); + } else if (interval.getStartTime() == 102L) { + Assert.assertTrue(interval.getEndTime() == 104L); + Assert.assertEquals(resource, Resource.newInstance(10 * 1024, 10)); + } else if (interval.getStartTime() == 104L) { + Assert.assertTrue(interval.getEndTime() == 105L); + Assert.assertEquals(resource, Resource.newInstance(5 * 1024, 5)); + } else if (interval.getStartTime() == 105L) { + Assert.assertTrue(interval.getEndTime() == 106L); + Assert.assertEquals(resource, Resource.newInstance(0 * 1024, 0)); + } else if (interval.getStartTime() == 106L) { + Assert.assertTrue(interval.getEndTime() == 107L); + Assert.assertEquals(resource, Resource.newInstance(5 * 1024, 5)); + } else { + Assert.fail(); + } + } + } + private Map<ReservationInterval, Resource> generateAllocation( int startTime, int[] alloc, boolean isStep) { Map<ReservationInterval, Resource> req = http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java index 50df8fe..f5625fb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java deleted file mode 100644 index d4a97ba..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSimpleCapacityReplanner.java +++ /dev/null @@ -1,162 +0,0 @@ -/******************************************************************************* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - *******************************************************************************/ -package org.apache.hadoop.yarn.server.resourcemanager.reservation; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.Map; -import java.util.TreeMap; - -import org.apache.hadoop.yarn.api.records.ReservationId; -import org.apache.hadoop.yarn.api.records.ReservationRequest; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; -import org.apache.hadoop.yarn.util.Clock; -import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; -import org.apache.hadoop.yarn.util.resource.ResourceCalculator; -import org.junit.Test; - -public class TestSimpleCapacityReplanner { - - @Test - public void testReplanningPlanCapacityLoss() throws PlanningException { - - Resource clusterCapacity = Resource.newInstance(100 * 1024, 10); - Resource minAlloc = Resource.newInstance(1024, 1); - Resource maxAlloc = Resource.newInstance(1024 * 8, 8); - - ResourceCalculator res = new DefaultResourceCalculator(); - long step = 1L; - Clock clock = mock(Clock.class); - ReservationAgent agent = mock(ReservationAgent.class); - - SharingPolicy policy = new NoOverCommitPolicy(); - policy.init("root.dedicated", null); - - QueueMetrics queueMetrics = mock(QueueMetrics.class); - - when(clock.getTime()).thenReturn(0L); - SimpleCapacityReplanner enf = new SimpleCapacityReplanner(clock); - - ReservationSchedulerConfiguration conf = - mock(ReservationSchedulerConfiguration.class); - when(conf.getEnforcementWindow(any(String.class))).thenReturn(6L); - - enf.init("blah", conf); - - // Initialize the plan with more resources - InMemoryPlan plan = - new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, - res, minAlloc, maxAlloc, "dedicated", enf, true, clock); - - // add reservation filling the plan (separating them 1ms, so we are sure - // s2 follows s1 on acceptance - long ts = System.currentTimeMillis(); - ReservationId r1 = ReservationId.newInstance(ts, 1); - int[] f5 = { 20, 20, 20, 20, 20 }; - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r1, null, "u3", - "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, - minAlloc))); - when(clock.getTime()).thenReturn(1L); - ReservationId r2 = ReservationId.newInstance(ts, 2); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r2, null, "u4", - "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, - minAlloc))); - when(clock.getTime()).thenReturn(2L); - ReservationId r3 = ReservationId.newInstance(ts, 3); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r3, null, "u5", - "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, - minAlloc))); - when(clock.getTime()).thenReturn(3L); - ReservationId r4 = ReservationId.newInstance(ts, 4); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r4, null, "u6", - "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, - minAlloc))); - when(clock.getTime()).thenReturn(4L); - ReservationId r5 = ReservationId.newInstance(ts, 5); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r5, null, "u7", - "dedicated", 0, 0 + f5.length, generateAllocation(0, f5), res, - minAlloc))); - - int[] f6 = { 50, 50, 50, 50, 50 }; - ReservationId r6 = ReservationId.newInstance(ts, 6); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r6, null, "u3", - "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, - minAlloc))); - when(clock.getTime()).thenReturn(6L); - ReservationId r7 = ReservationId.newInstance(ts, 7); - assertTrue(plan.toString(), - plan.addReservation(new InMemoryReservationAllocation(r7, null, "u4", - "dedicated", 10, 10 + f6.length, generateAllocation(10, f6), res, - minAlloc))); - - // remove some of the resources (requires replanning) - plan.setTotalCapacity(Resource.newInstance(70 * 1024, 70)); - - when(clock.getTime()).thenReturn(0L); - - // run the replanner - enf.plan(plan, null); - - // check which reservation are still present - assertNotNull(plan.getReservationById(r1)); - assertNotNull(plan.getReservationById(r2)); - assertNotNull(plan.getReservationById(r3)); - assertNotNull(plan.getReservationById(r6)); - assertNotNull(plan.getReservationById(r7)); - - // and which ones are removed - assertNull(plan.getReservationById(r4)); - assertNull(plan.getReservationById(r5)); - - // check resources at each moment in time no more exceed capacity - for (int i = 0; i < 20; i++) { - int tot = 0; - for (ReservationAllocation r : plan.getReservationsAtTime(i)) { - tot = r.getResourcesAtTime(i).getMemory(); - } - assertTrue(tot <= 70 * 1024); - } - } - - private Map<ReservationInterval, Resource> generateAllocation( - int startTime, int[] alloc) { - Map<ReservationInterval, Resource> req = - new TreeMap<ReservationInterval, Resource>(); - for (int i = 0; i < alloc.length; i++) { - req.put(new ReservationInterval(startTime + i, startTime + i + 1), - ReservationSystemUtil.toResource( - ReservationRequest.newInstance(Resource.newInstance(1024, 1), - alloc[i]))); - } - return req; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/d32b8b9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java new file mode 100644 index 0000000..9a1621a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestAlignedPlanner.java @@ -0,0 +1,820 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.apache.hadoop.yarn.api.records.ReservationDefinition; +import org.apache.hadoop.yarn.api.records.ReservationId; +import org.apache.hadoop.yarn.api.records.ReservationRequest; +import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter; +import org.apache.hadoop.yarn.api.records.ReservationRequests; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityOverTimePolicy; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryPlan; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.AlignedPlannerWithGreedy; +import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Before; +import org.junit.Test; +import org.mortbay.log.Log; + +public class TestAlignedPlanner { + + ReservationAgent agent; + InMemoryPlan plan; + Resource minAlloc = Resource.newInstance(1024, 1); + ResourceCalculator res = new DefaultResourceCalculator(); + Resource maxAlloc = Resource.newInstance(1024 * 8, 8); + Random rand = new Random(); + long step; + + @Test + public void testSingleReservationAccept() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario1(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 5 * step, // Job arrival time + 20 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(2048, 2), // Capability + 10, // Num containers + 5, // Concurrency + 10 * step) }, // Duration + ReservationRequestInterpreter.R_ORDER, "u1"); + + // Add reservation + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + + // CHECK: allocation was accepted + assertTrue("Agent-based allocation failed", reservationID != null); + assertTrue("Agent-based allocation failed", plan.getAllReservations() + .size() == numJobsInScenario + 1); + + // Get reservation + ReservationAllocation alloc1 = plan.getReservationById(reservationID); + + // Verify allocation + assertTrue(alloc1.toString(), + check(alloc1, 10 * step, 20 * step, 10, 2048, 2)); + + } + + @Test + public void testOrderNoGapImpossible() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10L, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ORDER_NO_GAP, "u1"); + + // Add reservation + try { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + fail(); + } catch (PlanningException e) { + // Expected failure + } + + // CHECK: allocation was not accepted + assertTrue("Agent-based allocation should have failed", plan + .getAllReservations().size() == numJobsInScenario); + + } + + @Test + public void testOrderNoGapImpossible2() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 13 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 10, // Num containers + 10, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ORDER_NO_GAP, "u1"); + + // Add reservation + try { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + fail(); + } catch (PlanningException e) { + // Expected failure + } + + // CHECK: allocation was not accepted + assertTrue("Agent-based allocation should have failed", plan + .getAllReservations().size() == numJobsInScenario); + + } + + @Test + public void testOrderImpossible() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + 2 * step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ORDER, "u1"); + + // Add reservation + try { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + fail(); + } catch (PlanningException e) { + // Expected failure + } + + // CHECK: allocation was not accepted + assertTrue("Agent-based allocation should have failed", plan + .getAllReservations().size() == numJobsInScenario); + + } + + @Test + public void testAnyImpossible() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + 3 * step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + 2 * step) }, // Duration + ReservationRequestInterpreter.R_ANY, "u1"); + + // Add reservation + try { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + fail(); + } catch (PlanningException e) { + // Expected failure + } + + // CHECK: allocation was not accepted + assertTrue("Agent-based allocation should have failed", plan + .getAllReservations().size() == numJobsInScenario); + + } + + @Test + public void testAnyAccept() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + 2 * step) }, // Duration + ReservationRequestInterpreter.R_ANY, "u1"); + + // Add reservation + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + + // CHECK: allocation was accepted + assertTrue("Agent-based allocation failed", reservationID != null); + assertTrue("Agent-based allocation failed", plan.getAllReservations() + .size() == numJobsInScenario + 1); + + // Get reservation + ReservationAllocation alloc1 = plan.getReservationById(reservationID); + + // Verify allocation + assertTrue(alloc1.toString(), + check(alloc1, 14 * step, 15 * step, 20, 1024, 1)); + + } + + @Test + public void testAllAccept() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Add reservation + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + + // CHECK: allocation was accepted + assertTrue("Agent-based allocation failed", reservationID != null); + assertTrue("Agent-based allocation failed", plan.getAllReservations() + .size() == numJobsInScenario + 1); + + // Get reservation + ReservationAllocation alloc1 = plan.getReservationById(reservationID); + + // Verify allocation + assertTrue(alloc1.toString(), + check(alloc1, 10 * step, 11 * step, 20, 1024, 1)); + assertTrue(alloc1.toString(), + check(alloc1, 14 * step, 15 * step, 20, 1024, 1)); + + } + + @Test + public void testAllImpossible() throws PlanningException { + + // Prepare basic plan + int numJobsInScenario = initializeScenario2(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + step), // Duration + ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + 2 * step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Add reservation + try { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + fail(); + } catch (PlanningException e) { + // Expected failure + } + + // CHECK: allocation was not accepted + assertTrue("Agent-based allocation should have failed", plan + .getAllReservations().size() == numJobsInScenario); + + } + + @Test + public void testUpdate() throws PlanningException { + + // Create flexible reservation + ReservationDefinition rrFlex = + createReservationDefinition( + 10 * step, // Job arrival time + 14 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 100, // Num containers + 1, // Concurrency + 2 * step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Create blocking reservation + ReservationDefinition rrBlock = + createReservationDefinition( + 10 * step, // Job arrival time + 11 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 100, // Num containers + 100, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Create reservation IDs + ReservationId flexReservationID = + ReservationSystemTestUtil.getNewReservationId(); + ReservationId blockReservationID = + ReservationSystemTestUtil.getNewReservationId(); + + // Add block, add flex, remove block, update flex + agent.createReservation(blockReservationID, "uBlock", plan, rrBlock); + agent.createReservation(flexReservationID, "uFlex", plan, rrFlex); + agent.deleteReservation(blockReservationID, "uBlock", plan); + agent.updateReservation(flexReservationID, "uFlex", plan, rrFlex); + + // CHECK: allocation was accepted + assertTrue("Agent-based allocation failed", flexReservationID != null); + assertTrue("Agent-based allocation failed", plan.getAllReservations() + .size() == 1); + + // Get reservation + ReservationAllocation alloc1 = plan.getReservationById(flexReservationID); + + // Verify allocation + assertTrue(alloc1.toString(), + check(alloc1, 10 * step, 14 * step, 50, 1024, 1)); + + } + + @Test + public void testImpossibleDuration() throws PlanningException { + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 15 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 20, // Num containers + 20, // Concurrency + 10 * step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Add reservation + try { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + fail(); + } catch (PlanningException e) { + // Expected failure + } + + // CHECK: allocation was not accepted + assertTrue("Agent-based allocation should have failed", plan + .getAllReservations().size() == 0); + + } + + @Test + public void testLoadedDurationIntervals() throws PlanningException { + + int numJobsInScenario = initializeScenario3(); + + // Create reservation + ReservationDefinition rr1 = + createReservationDefinition( + 10 * step, // Job arrival time + 13 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 80, // Num containers + 10, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Add reservation + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u1", plan, rr1); + + // CHECK: allocation was accepted + assertTrue("Agent-based allocation failed", reservationID != null); + assertTrue("Agent-based allocation failed", plan.getAllReservations() + .size() == numJobsInScenario + 1); + + // Get reservation + ReservationAllocation alloc1 = plan.getReservationById(reservationID); + + // Verify allocation + assertTrue(alloc1.toString(), + check(alloc1, 10 * step, 11 * step, 20, 1024, 1)); + assertTrue(alloc1.toString(), + check(alloc1, 11 * step, 12 * step, 20, 1024, 1)); + assertTrue(alloc1.toString(), + check(alloc1, 12 * step, 13 * step, 40, 1024, 1)); + } + + @Test + public void testCostFunction() throws PlanningException { + + // Create large memory reservation + ReservationDefinition rr7Mem1Core = + createReservationDefinition( + 10 * step, // Job arrival time + 11 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(7 * 1024, 1),// Capability + 1, // Num containers + 1, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1"); + + // Create reservation + ReservationDefinition rr6Mem6Cores = + createReservationDefinition( + 10 * step, // Job arrival time + 11 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(6 * 1024, 6),// Capability + 1, // Num containers + 1, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u2"); + + // Create reservation + ReservationDefinition rr = + createReservationDefinition( + 10 * step, // Job arrival time + 12 * step, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 1, // Num containers + 1, // Concurrency + step) }, // Duration + ReservationRequestInterpreter.R_ALL, "u3"); + + // Create reservation IDs + ReservationId reservationID1 = + ReservationSystemTestUtil.getNewReservationId(); + ReservationId reservationID2 = + ReservationSystemTestUtil.getNewReservationId(); + ReservationId reservationID3 = + ReservationSystemTestUtil.getNewReservationId(); + + // Add all + agent.createReservation(reservationID1, "u1", plan, rr7Mem1Core); + agent.createReservation(reservationID2, "u2", plan, rr6Mem6Cores); + agent.createReservation(reservationID3, "u3", plan, rr); + + // Get reservation + ReservationAllocation alloc3 = plan.getReservationById(reservationID3); + + assertTrue(alloc3.toString(), + check(alloc3, 10 * step, 11 * step, 0, 1024, 1)); + assertTrue(alloc3.toString(), + check(alloc3, 11 * step, 12 * step, 1, 1024, 1)); + + } + + @Test + public void testFromCluster() throws PlanningException { + + // int numJobsInScenario = initializeScenario3(); + + List<ReservationDefinition> list = new ArrayList<ReservationDefinition>(); + + // Create reservation + list.add(createReservationDefinition( + 1425716392178L, // Job arrival time + 1425722262791L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 7, // Num containers + 1, // Concurrency + 587000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u1")); + + list.add(createReservationDefinition( + 1425716406178L, // Job arrival time + 1425721255841L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 6, // Num containers + 1, // Concurrency + 485000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u2")); + + list.add(createReservationDefinition( + 1425716399178L, // Job arrival time + 1425723780138L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 6, // Num containers + 1, // Concurrency + 738000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u3")); + + list.add(createReservationDefinition( + 1425716437178L, // Job arrival time + 1425722968378L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 7, // Num containers + 1, // Concurrency + 653000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u4")); + + list.add(createReservationDefinition( + 1425716406178L, // Job arrival time + 1425721926090L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 6, // Num containers + 1, // Concurrency + 552000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u5")); + + list.add(createReservationDefinition( + 1425716379178L, // Job arrival time + 1425722238553L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 6, // Num containers + 1, // Concurrency + 586000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u6")); + + list.add(createReservationDefinition( + 1425716407178L, // Job arrival time + 1425722908317L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 7, // Num containers + 1, // Concurrency + 650000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u7")); + + list.add(createReservationDefinition( + 1425716452178L, // Job arrival time + 1425722841562L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 6, // Num containers + 1, // Concurrency + 639000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u8")); + + list.add(createReservationDefinition( + 1425716384178L, // Job arrival time + 1425721766129L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 7, // Num containers + 1, // Concurrency + 538000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u9")); + + list.add(createReservationDefinition( + 1425716437178L, // Job arrival time + 1425722507886L, // Job deadline + new ReservationRequest[] { ReservationRequest.newInstance( + Resource.newInstance(1024, 1), // Capability + 5, // Num containers + 1, // Concurrency + 607000) }, // Duration + ReservationRequestInterpreter.R_ALL, "u10")); + + // Add reservation + int i = 1; + for (ReservationDefinition rr : list) { + ReservationId reservationID = + ReservationSystemTestUtil.getNewReservationId(); + agent.createReservation(reservationID, "u" + Integer.toString(i), plan, + rr); + ++i; + } + + // CHECK: allocation was accepted + assertTrue("Agent-based allocation failed", plan.getAllReservations() + .size() == list.size()); + + } + + @Before + public void setup() throws Exception { + + // Initialize random seed + long seed = rand.nextLong(); + rand.setSeed(seed); + Log.info("Running with seed: " + seed); + + // Set cluster parameters + long timeWindow = 1000000L; + int capacityMem = 100 * 1024; + int capacityCores = 100; + step = 60000L; + + Resource clusterCapacity = Resource.newInstance(capacityMem, capacityCores); + + // Set configuration + ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil(); + String reservationQ = testUtil.getFullReservationQueueName(); + float instConstraint = 100; + float avgConstraint = 100; + + ReservationSchedulerConfiguration conf = + ReservationSystemTestUtil.createConf(reservationQ, timeWindow, + instConstraint, avgConstraint); + + CapacityOverTimePolicy policy = new CapacityOverTimePolicy(); + policy.init(reservationQ, conf); + + QueueMetrics queueMetrics = mock(QueueMetrics.class); + + // Set planning agent + agent = new AlignedPlannerWithGreedy(); + + // Create Plan + plan = + new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step, + res, minAlloc, maxAlloc, "dedicated", null, true); + } + + private int initializeScenario1() throws PlanningException { + + // insert in the reservation a couple of controlled reservations, to create + // conditions for assignment that are non-empty + + addFixedAllocation(0L, step, new int[] { 10, 10, 20, 20, 20, 10, 10 }); + + System.out.println("--------BEFORE AGENT----------"); + System.out.println(plan.toString()); + System.out.println(plan.toCumulativeString()); + + return 1; + + } + + private int initializeScenario2() throws PlanningException { + + // insert in the reservation a couple of controlled reservations, to create + // conditions for assignment that are non-empty + + addFixedAllocation(11 * step, step, new int[] { 90, 90, 90 }); + + System.out.println("--------BEFORE AGENT----------"); + System.out.println(plan.toString()); + System.out.println(plan.toCumulativeString()); + + return 1; + + } + + private int initializeScenario3() throws PlanningException { + + // insert in the reservation a couple of controlled reservations, to create + // conditions for assignment that are non-empty + + addFixedAllocation(10 * step, step, new int[] { 70, 80, 60 }); + + System.out.println("--------BEFORE AGENT----------"); + System.out.println(plan.toString()); + System.out.println(plan.toCumulativeString()); + + return 1; + + } + + private void addFixedAllocation(long start, long step, int[] f) + throws PlanningException { + + assertTrue(plan.toString(), + plan.addReservation(new InMemoryReservationAllocation( + ReservationSystemTestUtil.getNewReservationId(), null, + "user_fixed", "dedicated", start, start + f.length * step, + ReservationSystemTestUtil.generateAllocation(start, step, f), res, + minAlloc))); + + } + + private ReservationDefinition createReservationDefinition(long arrival, + long deadline, ReservationRequest[] reservationRequests, + ReservationRequestInterpreter rType, String username) { + + return ReservationDefinition.newInstance(arrival, deadline, + ReservationRequests.newInstance(Arrays.asList(reservationRequests), + rType), username); + + } + + private boolean check(ReservationAllocation alloc, long start, long end, + int containers, int mem, int cores) { + + Resource expectedResources = + Resource.newInstance(mem * containers, cores * containers); + + // Verify that all allocations in [start,end) equal containers * (mem,cores) + for (long i = start; i < end; i++) { + if (!Resources.equals(alloc.getResourcesAtTime(i), expectedResources)) { + return false; + } + } + return true; + + } + +}