http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java index 3508ab4..ad3413f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java @@ -23,17 +23,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; -import java.util.Map; -import org.apache.commons.lang.time.DateUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; @@ -43,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage; +import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; import org.apache.log4j.Level; @@ -84,13 +80,13 @@ public class TestContainerResourceUsage { RMAppMetrics rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( - "Before app submittion, memory seconds should have been 0 but was " - + rmAppMetrics.getMemorySeconds(), - rmAppMetrics.getMemorySeconds() == 0); + "Before app submission, memory seconds should have been 0 but was " + + rmAppMetrics.getGuaranteedMemorySeconds(), + rmAppMetrics.getGuaranteedMemorySeconds() == 0); Assert.assertTrue( "Before app submission, vcore seconds should have been 0 but was " - + rmAppMetrics.getVcoreSeconds(), - rmAppMetrics.getVcoreSeconds() == 0); + + rmAppMetrics.getGuaranteedVcoreSeconds(), + rmAppMetrics.getGuaranteedVcoreSeconds() == 0); RMAppAttempt attempt0 = app0.getCurrentAppAttempt(); @@ -105,7 +101,8 @@ public class TestContainerResourceUsage { // Allow metrics to accumulate. int sleepInterval = 1000; int cumulativeSleepTime = 0; - while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { + while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0 + && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; } @@ -113,27 +110,35 @@ public class TestContainerResourceUsage { rmAppMetrics = app0.getRMAppMetrics(); Assert.assertTrue( "While app is running, memory seconds should be >0 but is " - + rmAppMetrics.getMemorySeconds(), - rmAppMetrics.getMemorySeconds() > 0); + + rmAppMetrics.getGuaranteedMemorySeconds(), + rmAppMetrics.getGuaranteedMemorySeconds() > 0); Assert.assertTrue( "While app is running, vcore seconds should be >0 but is " - + rmAppMetrics.getVcoreSeconds(), - rmAppMetrics.getVcoreSeconds() > 0); + + rmAppMetrics.getGuaranteedVcoreSeconds(), + rmAppMetrics.getGuaranteedVcoreSeconds() > 0); MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer); rmAppMetrics = app0.getRMAppMetrics(); - Assert.assertEquals("Unexpected MemorySeconds value", - ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds()); - Assert.assertEquals("Unexpected VcoreSeconds value", - ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds()); + Assert.assertEquals("Unexpected GuaranteedMemorySeconds value", + ru.getGuaranteedMemorySeconds(), + rmAppMetrics.getGuaranteedMemorySeconds()); + Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value", + ru.getGuaranteedVcoreSeconds(), + rmAppMetrics.getGuaranteedVcoreSeconds()); + Assert.assertEquals("Unexpected OpportunisticMemorySeconds value", + ru.getOpportunisticMemorySeconds(), + rmAppMetrics.getOpportunisticMemorySeconds()); + Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value", + ru.getOpportunisticVcoreSeconds(), + rmAppMetrics.getOpportunisticVcoreSeconds()); rm.stop(); } - @Test (timeout = 120000) + @Test () public void testUsageWithMultipleContainersAndRMRestart() throws Exception { // Set max attempts to 1 so that when the first attempt fails, the app // won't try to start a new one. @@ -191,7 +196,7 @@ public class TestContainerResourceUsage { // Allow metrics to accumulate. int sleepInterval = 1000; int cumulativeSleepTime = 0; - while (app0.getRMAppMetrics().getMemorySeconds() <= 0 + while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0 && cumulativeSleepTime < 5000) { Thread.sleep(sleepInterval); cumulativeSleepTime += sleepInterval; @@ -216,19 +221,29 @@ public class TestContainerResourceUsage { rm0.waitForState(nm, cId, RMContainerState.COMPLETED); // Check that the container metrics match those from the app usage report. - long memorySeconds = 0; - long vcoreSeconds = 0; + long guaranteedMemorySeconds = 0; + long guaranteedVcoreSeconds = 0; + long opportunisticMemorySeconds = 0; + long opportunisticVcoreSeconds = 0; for (RMContainer c : rmContainers) { AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); } RMAppMetrics metricsBefore = app0.getRMAppMetrics(); - Assert.assertEquals("Unexpected MemorySeconds value", - memorySeconds, metricsBefore.getMemorySeconds()); - Assert.assertEquals("Unexpected VcoreSeconds value", - vcoreSeconds, metricsBefore.getVcoreSeconds()); + Assert.assertEquals("Unexpected GuaranteedMemorySeconds value", + guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds()); + Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value", + guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds()); + Assert.assertEquals("Unexpected OpportunisticMemorySeconds value", + opportunisticMemorySeconds, + metricsBefore.getOpportunisticMemorySeconds()); + Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value", + opportunisticVcoreSeconds, + metricsBefore.getOpportunisticVcoreSeconds()); // create new RM to represent RM restart. Load up the state store. MockRM rm1 = new MockRM(conf, memStore); @@ -238,10 +253,22 @@ public class TestContainerResourceUsage { // Compare container resource usage metrics from before and after restart. RMAppMetrics metricsAfter = app0After.getRMAppMetrics(); - Assert.assertEquals("Vcore seconds were not the same after RM Restart", - metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds()); - Assert.assertEquals("Memory seconds were not the same after RM Restart", - metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds()); + Assert.assertEquals( + "Guaranteed vcore seconds were not the same after RM Restart", + metricsBefore.getGuaranteedVcoreSeconds(), + metricsAfter.getGuaranteedVcoreSeconds()); + Assert.assertEquals( + "Guaranteed memory seconds were not the same after RM Restart", + metricsBefore.getGuaranteedMemorySeconds(), + metricsAfter.getGuaranteedMemorySeconds()); + Assert.assertEquals( + "Opportunistic vcore seconds were not the same after RM Restart", + metricsBefore.getOpportunisticVcoreSeconds(), + metricsAfter.getOpportunisticVcoreSeconds()); + Assert.assertEquals( + "Opportunistic memory seconds were not the same after RM Restart", + metricsBefore.getOpportunisticMemorySeconds(), + metricsAfter.getOpportunisticMemorySeconds()); rm0.stop(); rm0.close(); @@ -264,10 +291,8 @@ public class TestContainerResourceUsage { MockRM rm = new MockRM(conf); rm.start(); - RMApp app = - rm.submitApp(200, "name", "user", - new HashMap<ApplicationAccessType, String>(), false, "default", -1, - null, "MAPREDUCE", false, keepRunningContainers); + RMApp app = rm.submitApp(200, "name", "user", new HashMap<>(), false, + "default", -1, null, "MAPREDUCE", false, keepRunningContainers); MockNM nm = new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService()); nm.registerNode(); @@ -275,18 +300,17 @@ public class TestContainerResourceUsage { MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm); int NUM_CONTAINERS = 1; // allocate NUM_CONTAINERS containers - am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, - new ArrayList<ContainerId>()); + am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>()); nm.nodeHeartbeat(true); // wait for containers to be allocated. List<Container> containers = - am0.allocate(new ArrayList<ResourceRequest>(), - new ArrayList<ContainerId>()).getAllocatedContainers(); + am0.allocate(new ArrayList<>(), new ArrayList<>()) + .getAllocatedContainers(); while (containers.size() != NUM_CONTAINERS) { nm.nodeHeartbeat(true); - containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(), - new ArrayList<ContainerId>()).getAllocatedContainers()); + containers.addAll(am0.allocate(new ArrayList<>(), + new ArrayList<>()).getAllocatedContainers()); Thread.sleep(200); } @@ -294,26 +318,27 @@ public class TestContainerResourceUsage { ContainerId containerId2 = ContainerId.newContainerId(am0.getApplicationAttemptId(), 2); nm.nodeHeartbeat(am0.getApplicationAttemptId(), - containerId2.getContainerId(), ContainerState.RUNNING); + containerId2.getContainerId(), ContainerState.RUNNING); rm.waitForState(nm, containerId2, RMContainerState.RUNNING); // Capture the containers here so the metrics can be calculated after the // app has completed. - Collection<RMContainer> rmContainers = - rm.scheduler - .getSchedulerAppInfo(am0.getApplicationAttemptId()) - .getLiveContainers(); + Collection<RMContainer> rmContainers = rm.scheduler + .getSchedulerAppInfo(am0.getApplicationAttemptId()) + .getLiveContainers(); // fail the first app attempt by sending CONTAINER_FINISHED event without // registering. ContainerId amContainerId = app.getCurrentAppAttempt().getMasterContainer().getId(); nm.nodeHeartbeat(am0.getApplicationAttemptId(), - amContainerId.getContainerId(), ContainerState.COMPLETE); + amContainerId.getContainerId(), ContainerState.COMPLETE); rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED); rm.drainEvents(); - long memorySeconds = 0; - long vcoreSeconds = 0; + long guaranteedMemorySeconds = 0; + long guaranteedVcoreSeconds = 0; + long opportunisticMemorySeconds = 0; + long opportunisticVcoreSeconds = 0; // Calculate container usage metrics for first attempt. if (keepRunningContainers) { @@ -321,8 +346,10 @@ public class TestContainerResourceUsage { for (RMContainer c : rmContainers) { if (c.getContainerId().equals(amContainerId)) { AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); } else { // The remaining container should be RUNNING. Assert.assertTrue("After first attempt failed, remaining container " @@ -336,8 +363,10 @@ public class TestContainerResourceUsage { for (RMContainer c : rmContainers) { waitforContainerCompletion(rm, nm, amContainerId, c); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); } } @@ -355,14 +384,12 @@ public class TestContainerResourceUsage { am1.registerAppAttempt(); rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING); // allocate NUM_CONTAINERS containers - am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, - new ArrayList<ContainerId>()); + am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>()); nm.nodeHeartbeat(true); // wait for containers to be allocated. - containers = - am1.allocate(new ArrayList<ResourceRequest>(), - new ArrayList<ContainerId>()).getAllocatedContainers(); + containers = am1.allocate(new ArrayList<>(), new ArrayList<>()) + .getAllocatedContainers(); while (containers.size() != NUM_CONTAINERS) { nm.nodeHeartbeat(true); containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(), @@ -388,19 +415,26 @@ public class TestContainerResourceUsage { for (RMContainer c : rmContainers) { waitforContainerCompletion(rm, nm, amContainerId, c); AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c); - memorySeconds += ru.getMemorySeconds(); - vcoreSeconds += ru.getVcoreSeconds(); + guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds(); + guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds(); + opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds(); + opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds(); } RMAppMetrics rmAppMetrics = app.getRMAppMetrics(); - Assert.assertEquals("Unexpected MemorySeconds value", - memorySeconds, rmAppMetrics.getMemorySeconds()); - Assert.assertEquals("Unexpected VcoreSeconds value", - vcoreSeconds, rmAppMetrics.getVcoreSeconds()); + Assert.assertEquals("Unexpected GuaranteedMemorySeconds value", + guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds()); + Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value", + guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds()); + Assert.assertEquals("Unexpected OpportunisticMemorySeconds value", + opportunisticMemorySeconds, + rmAppMetrics.getOpportunisticMemorySeconds()); + Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value", + opportunisticVcoreSeconds, + rmAppMetrics.getOpportunisticVcoreSeconds()); rm.stop(); - return; } private void waitforContainerCompletion(MockRM rm, MockNM nm, @@ -419,16 +453,10 @@ public class TestContainerResourceUsage { private AggregateAppResourceUsage calculateContainerResourceMetrics( RMContainer rmContainer) { - Resource resource = rmContainer.getContainer().getResource(); - long usedMillis = - rmContainer.getFinishTime() - rmContainer.getCreationTime(); - long memorySeconds = resource.getMemorySize() - * usedMillis / DateUtils.MILLIS_PER_SECOND; - long vcoreSeconds = resource.getVirtualCores() - * usedMillis / DateUtils.MILLIS_PER_SECOND; - Map<String, Long> map = new HashMap<>(); - map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); - map.put(ResourceInformation.VCORES.getName(), vcoreSeconds); - return new AggregateAppResourceUsage(map); + ContainerResourceUsageReport resourceUsageReport = + rmContainer.getResourceUsageReport(); + return new AggregateAppResourceUsage( + resourceUsageReport.getGuaranteedResourceUsageSecondsMap(), + resourceUsageReport.getOpportunisticResourceSecondsMap()); } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java index 72de27c..3c2db3e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java @@ -193,7 +193,7 @@ public abstract class MockAsm extends MockApps { @Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), - new HashMap<>()); + new HashMap<>(), new HashMap<>()); } @Override @@ -353,7 +353,7 @@ public abstract class MockAsm extends MockApps { ApplicationResourceUsageReport usageReport = ApplicationResourceUsageReport .newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0, - new HashMap<>()); + new HashMap<>(), null, new HashMap<>()); ApplicationReport report = ApplicationReport.newInstance( getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java index 5b0c34f..cdfd361 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java @@ -479,7 +479,7 @@ public class TestCombinedSystemMetricsPublisher { preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(rmApp.getRMAppMetrics()).thenReturn( new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap, - preemptedMap)); + preemptedMap, new HashMap<>())); when(rmApp.getApplicationTags()).thenReturn( Collections.<String> emptySet()); ApplicationSubmissionContext appSubmissionContext = http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java index 2287617..c9e2d04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java @@ -219,13 +219,25 @@ public class TestSystemMetricsPublisher { entity.getOtherInfo().get( ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)); Assert.assertEquals( - app.getRMAppMetrics().getMemorySeconds(), + app.getRMAppMetrics().getGuaranteedMemorySeconds(), Long.parseLong(entity.getOtherInfo() - .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString())); + .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS) + .toString())); + Assert.assertEquals( + app.getRMAppMetrics().getGuaranteedVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS) + .toString())); Assert.assertEquals( - app.getRMAppMetrics().getVcoreSeconds(), + app.getRMAppMetrics().getOpportunisticMemorySeconds(), Long.parseLong(entity.getOtherInfo() - .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString())); + .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS) + .toString())); + Assert.assertEquals( + app.getRMAppMetrics().getOpportunisticVcoreSeconds(), + Long.parseLong(entity.getOtherInfo() + .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS) + .toString())); Assert.assertEquals( app.getRMAppMetrics().getPreemptedMemorySeconds(), Long.parseLong(entity.getOtherInfo() @@ -517,7 +529,8 @@ public class TestSystemMetricsPublisher { .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()) - .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap)); + .thenReturn(new RMAppMetrics(null, 0, 0, + resourceMap, preemptedMap, new HashMap<>())); Set<String> appTags = new HashSet<String>(); appTags.add("test"); appTags.add("tags"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java index 68bb325..7aa1a60 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java @@ -221,7 +221,7 @@ public class TestSystemMetricsPublisherForV2 { File appFile = new File(outputDirApp, timelineServiceFileName); Assert.assertTrue(appFile.exists()); verifyEntity( - appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0); + appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0); } @Test(timeout = 10000) @@ -365,14 +365,14 @@ public class TestSystemMetricsPublisherForV2 { when(app.getCurrentAppAttempt()).thenReturn(appAttempt); when(app.getFinalApplicationStatus()) .thenReturn(FinalApplicationStatus.UNDEFINED); - Map<String, Long> resourceSecondsMap = new HashMap<>(); - resourceSecondsMap + Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(); + guaranteedResourceSecondsMap .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE); - resourceSecondsMap + guaranteedResourceSecondsMap .put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE); when(app.getRMAppMetrics()).thenReturn( - new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap, - new HashMap<>())); + new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, + guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>())); when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet()); ApplicationSubmissionContext appSubmissionContext = mock(ApplicationSubmissionContext.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java index dbb2148..5f01c39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java @@ -196,7 +196,8 @@ public class RMStateStoreTestBase { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); + .thenReturn(new AggregateAppResourceUsage( + new HashMap<>(), new HashMap<>())); dispatcher.attemptId = attemptId; store.storeNewApplicationAttempt(mockAttempt); waitNotify(dispatcher); @@ -294,7 +295,8 @@ public class RMStateStoreTestBase { when(mockRemovedAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); + .thenReturn(new AggregateAppResourceUsage( + new HashMap<>(), new HashMap<>())); attempts.put(attemptIdRemoved, mockRemovedAttempt); store.removeApplication(mockRemovedApp); @@ -371,7 +373,8 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 100, - oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>()); + oldAttemptState.getFinishTime(), new HashMap<>(), + new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not @@ -396,7 +399,8 @@ public class RMStateStoreTestBase { oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, 111, - oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>()); + oldAttemptState.getFinishTime(), new HashMap<>(), + new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(dummyAttempt); // let things settle down http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java index 0a1b152..16b3abf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java @@ -547,7 +547,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { when(mockAttempt.getRMAppAttemptMetrics()) .thenReturn(mockRmAppAttemptMetrics); when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage()) - .thenReturn(new AggregateAppResourceUsage(new HashMap<>())); + .thenReturn(new AggregateAppResourceUsage(new HashMap<>(), + new HashMap<>(0))); store.storeNewApplicationAttempt(mockAttempt); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -559,7 +560,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { store.getCredentialsFromAppAttempt(mockAttempt), startTime, RMAppAttemptState.FINISHED, "testUrl", "test", FinalApplicationStatus.SUCCEEDED, 100, - finishTime, new HashMap<>(), new HashMap<>()); + finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>()); store.updateApplicationAttemptState(newAttemptState); assertEquals("RMStateStore should have been in fenced state", true, store.isFencedState()); @@ -796,20 +797,33 @@ public class TestZKRMStateStore extends RMStateStoreTestBase { private static ApplicationAttemptStateData createFinishedAttempt( ApplicationAttemptId attemptId, Container container, long startTime, int amExitStatus) { - Map<String, Long> resourceSecondsMap = new HashMap<>(); - Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); - resourceSecondsMap - .put(ResourceInformation.MEMORY_MB.getName(), 0L); - resourceSecondsMap - .put(ResourceInformation.VCORES.getName(), 0L); - preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), - 0L); - preemptedResoureSecondsMap - .put(ResourceInformation.VCORES.getName(), 0L); + Map<String, Long> guaranteedResourceSecondsMap = + new HashMap<String, Long>() { + { + put(ResourceInformation.MEMORY_MB.getName(), 0L); + put(ResourceInformation.VCORES.getName(), 0L); + } + }; + Map<String, Long> opportunisticResourceSecondsMap = + new HashMap<String, Long>() { + { + put(ResourceInformation.MEMORY_MB.getName(), 0L); + put(ResourceInformation.VCORES.getName(), 0L); + } + }; + Map<String, Long> preemptedResoureSecondsMap = + new HashMap<String, Long>() { + { + put(ResourceInformation.MEMORY_MB.getName(), 0L); + put(ResourceInformation.VCORES.getName(), 0L); + } + }; + return ApplicationAttemptStateData.newInstance(attemptId, container, null, startTime, RMAppAttemptState.FINISHED, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED, - amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap); + amExitStatus, 0, guaranteedResourceSecondsMap, + preemptedResoureSecondsMap, opportunisticResourceSecondsMap); } private ApplicationAttemptId storeAttempt(RMStateStore store, http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index 920bd4f..dde74a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod; import org.apache.hadoop.yarn.MockApps; import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; @@ -243,7 +242,7 @@ public class TestRMAppTransitions { ResourceScheduler resourceScheduler = mock(ResourceScheduler.class); doReturn(null).when(resourceScheduler) - .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()); + .getAppActiveResourceUsageReport(Matchers.any()); doReturn(resourceScheduler).when(rmContext).getScheduler(); doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext) @@ -1249,7 +1248,8 @@ public class TestRMAppTransitions { assertAppState(RMAppState.NEW, app); ApplicationReport report = app.createAndGetApplicationReport(null, true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); - Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT); + Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT, + report.getApplicationResourceUsageReport()); report = app.createAndGetApplicationReport("clientuser", true); Assert.assertNotNull(report.getApplicationResourceUsageReport()); Assert.assertTrue("bad proxy url for app", http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java index 4a5c671..550e24d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java @@ -299,10 +299,12 @@ public class TestRMAppAttemptTransitions { ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); - when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); - when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); + when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L); + when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L); + when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L); + when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L); when(resourceScheduler - .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) + .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any())) .thenReturn(appResUsgRpt); spyRMContext = spy(rmContext); Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); @@ -762,10 +764,12 @@ public class TestRMAppAttemptTransitions { ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId(); ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); - when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L); - when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L); - when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class))) - .thenReturn(appResUsgRpt); + when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L); + when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L); + when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L); + when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L); + when(scheduler.getAppActiveResourceUsageReport( + any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt); // start and finish the attempt Container amContainer = allocateApplicationAttempt(); @@ -777,12 +781,14 @@ public class TestRMAppAttemptTransitions { // expect usage stats to come from the scheduler report ApplicationResourceUsageReport report = applicationAttempt.getApplicationResourceUsageReport(); - Assert.assertEquals(123456L, report.getMemorySeconds()); - Assert.assertEquals(55544L, report.getVcoreSeconds()); + Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds()); + Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds()); + Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds()); + Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds()); // finish app attempt and remove it from scheduler - when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L); - when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L); + when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L); + when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L); sendAttemptUpdateSavedEvent(applicationAttempt); NodeId anyNodeId = NodeId.newInstance("host", 1234); applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent( @@ -793,8 +799,8 @@ public class TestRMAppAttemptTransitions { when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null); report = applicationAttempt.getApplicationResourceUsageReport(); - Assert.assertEquals(223456, report.getMemorySeconds()); - Assert.assertEquals(75544, report.getVcoreSeconds()); + Assert.assertEquals(223456, report.getGuaranteedMemorySeconds()); + Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds()); } @Test http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 89a92fb..cc71990 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2841,7 +2841,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { Thread.sleep(1000); AllocateResponse allocResponse = am1.schedule(); ApplicationResourceUsageReport report = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( attempt1.getAppAttemptId()); Assert.assertEquals(0, allocResponse.getAllocatedContainers().size()); Assert.assertEquals(0, report.getNumReservedContainers()); @@ -2854,7 +2854,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { allocResponse = am1.schedule(); } report = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( attempt1.getAppAttemptId()); Assert.assertEquals(1, allocResponse.getAllocatedContainers().size()); Assert.assertEquals(0, report.getNumReservedContainers()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index b4ebd15..a3ae7bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -223,23 +223,23 @@ public class TestCapacitySchedulerNodeLabelUpdate { RMApp app1 = rm.submitApp(GB, "app", "user", null, "a"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3); ApplicationResourceUsageReport appResourceUsageReport = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( am1.getApplicationAttemptId()); - Assert.assertEquals(1024, appResourceUsageReport.getUsedResources() - .getMemorySize()); - Assert.assertEquals(1, appResourceUsageReport.getUsedResources() - .getVirtualCores()); + Assert.assertEquals(1024, appResourceUsageReport + .getGuaranteedResourcesUsed().getMemorySize()); + Assert.assertEquals(1, appResourceUsageReport + .getGuaranteedResourcesUsed().getVirtualCores()); // request a container. am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x"); containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2); rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED); appResourceUsageReport = - rm.getResourceScheduler().getAppResourceUsageReport( + rm.getResourceScheduler().getAppActiveResourceUsageReport( am1.getApplicationAttemptId()); - Assert.assertEquals(2048, appResourceUsageReport.getUsedResources() - .getMemorySize()); - Assert.assertEquals(2, appResourceUsageReport.getUsedResources() - .getVirtualCores()); + Assert.assertEquals(2048, appResourceUsageReport + .getGuaranteedResourcesUsed().getMemorySize()); + Assert.assertEquals(2, appResourceUsageReport + .getGuaranteedResourcesUsed().getVirtualCores()); LeafQueue queue = (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler()) .getQueue("a"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java index f88ac8b..de97ea4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java @@ -66,7 +66,7 @@ public class TestAppPage { RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(), - new HashMap<>()); + new HashMap<>(), new HashMap<>()); when(app.getRMAppMetrics()).thenReturn(appMetrics); // initialize RM Context, and create RMApp, without creating RMAppAttempt http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java index b125608..93efe23 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java @@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Token; @@ -259,7 +258,7 @@ public class TestRMWebApp { app.getDiagnostics().toString(), (String) null, app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), - (ApplicationResourceUsageReport) null, app.getTrackingUrl(), + null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null); appReports.add(appReport); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java index cea0088..87846c7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java @@ -138,7 +138,7 @@ public class TestRMWebAppFairScheduler { @Override public RMAppMetrics getRMAppMetrics() { return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, - new HashMap<>(), new HashMap<>()); + new HashMap<>(), new HashMap<>(), new HashMap<>()); } @Override public YarnApplicationState createApplicationState() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java index a381ed4..754d8b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java @@ -1603,7 +1603,7 @@ public class TestRMWebServicesApps extends JerseyTestBase { public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs) throws JSONException, Exception { - int expectedNumberOfElements = 39 + (hasResourceReqs ? 2 : 0); + int expectedNumberOfElements = 44 + (hasResourceReqs ? 2 : 0); String appNodeLabelExpression = null; String amNodeLabelExpression = null; if (app.getApplicationSubmissionContext() http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java index 40bdbd8..6f1ee3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java @@ -339,12 +339,22 @@ public final class RouterWebServiceUtil { am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB()); am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores()); + am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() + + uam.getAllocatedOpportunisticMB()); + am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() + + uam.getAllocatedOpportunisticVCores()); am.setReservedMB(am.getReservedMB() + uam.getReservedMB()); am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB()); am.setRunningContainers( am.getRunningContainers() + uam.getRunningContainers()); - am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds()); - am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds()); + am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() + + uam.getGuaranteedMemorySeconds()); + am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() + + uam.getGuaranteedVcoreSeconds()); + am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() + + uam.getOpportunisticVcoreSeconds()); + am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() + + uam.getOpportunisticMemorySeconds()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/eca4df88/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java index edf3804..1a5c42e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java @@ -249,8 +249,10 @@ public class TestRouterWebServiceUtil { Assert.assertEquals(value * 3, app.getReservedMB()); Assert.assertEquals(value * 3, app.getReservedVCores()); Assert.assertEquals(value * 3, app.getRunningContainers()); - Assert.assertEquals(value * 3, app.getMemorySeconds()); - Assert.assertEquals(value * 3, app.getVcoreSeconds()); + Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds()); + Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds()); + Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds()); + Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds()); Assert.assertEquals(3, app.getResourceRequests().size()); } @@ -258,11 +260,15 @@ public class TestRouterWebServiceUtil { am.getResourceRequests().add(new ResourceRequestInfo()); am.setAllocatedMB(value); am.setAllocatedVCores(value); + am.setAllocatedOpportunisticVCores(value); + am.setAllocatedOpportunisticMB(value); am.setReservedMB(value); am.setReservedVCores(value); am.setRunningContainers(value); - am.setMemorySeconds(value); - am.setVcoreSeconds(value); + am.setGuaranteedMemorySeconds(value); + am.setGuaranteedVcoreSeconds(value); + am.setOpportunisticMemorySeconds(value); + am.setOpportunisticVcoreSeconds(value); } /** --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
