Repository: hadoop Updated Branches: refs/heads/trunk 3f6195045 -> e5287a4fe
YARN-8824. App Nodelabel missed after RM restart for finished apps. Contributed by Bibin A Chundatt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5287a4f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5287a4f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5287a4f Branch: refs/heads/trunk Commit: e5287a4fe0bb03d929f066fc50eb0e7bd74bb759 Parents: 3f61950 Author: Rohith Sharma K S <[email protected]> Authored: Wed Sep 26 12:30:26 2018 +0530 Committer: Rohith Sharma K S <[email protected]> Committed: Wed Sep 26 12:30:26 2018 +0530 ---------------------------------------------------------------------- .../resourcemanager/recovery/RMStateStore.java | 1 + .../resourcemanager/MockMemoryRMStateStore.java | 1 + .../yarn/server/resourcemanager/MockRM.java | 31 +++++++++++++ .../server/resourcemanager/TestRMRestart.java | 49 ++++++++++++++++++++ 4 files changed, 82 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java index a0b10b1..ccd6fc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java @@ -307,6 +307,7 @@ public abstract class RMStateStore extends AbstractService { context.setApplicationTags(srcCtx.getApplicationTags()); context.setApplicationType(srcCtx.getApplicationType()); context.setUnmanagedAM(srcCtx.getUnmanagedAM()); + context.setNodeLabelExpression(srcCtx.getNodeLabelExpression()); ContainerLaunchContextPBImpl amContainerSpec = new ContainerLaunchContextPBImpl(); amContainerSpec.setApplicationACLs( http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java index 698f1c6..ef9606f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java @@ -105,6 +105,7 @@ public class MockMemoryRMStateStore extends MemoryRMStateStore { oldAppSubCtxt.getAMContainerResourceRequests()); context.setLogAggregationContext(oldAppSubCtxt.getLogAggregationContext()); context.setApplicationType(oldAppSubCtxt.getApplicationType()); + context.setNodeLabelExpression(oldAppSubCtxt.getNodeLabelExpression()); this.appSubCtxtCopy.put(oldAppSubCtxt.getApplicationId(), context); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index 6a024f5..df86f28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -722,6 +722,17 @@ public class MockRM extends ResourceManager { amResourceRequests.get(0).getNodeLabelExpression(), null, null); } + public RMApp submitApp(List<ResourceRequest> amResourceRequests, + String appNodeLabel) throws Exception { + return submitApp(amResourceRequests, "app1", "user", null, false, null, + super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, + YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true, + false, false, null, 0, null, true, + amResourceRequests.get(0).getPriority(), + amResourceRequests.get(0).getNodeLabelExpression(), null, null, null, + appNodeLabel); + } + public RMApp submitApp(Resource capability, String name, String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue, int maxAppAttempts, Credentials ts, String appType, @@ -771,6 +782,23 @@ public class MockRM extends ResourceManager { boolean cancelTokensWhenComplete, Priority priority, String amLabel, Map<ApplicationTimeoutType, Long> applicationTimeouts, ByteBuffer tokensConf, Set<String> applicationTags) throws Exception { + return submitApp(amResourceRequests, name, user, acls, unmanaged, queue, + maxAppAttempts, ts, appType, waitForAccepted, keepContainers, + isAppIdProvided, applicationId, attemptFailuresValidityInterval, + logAggregationContext, cancelTokensWhenComplete, priority, amLabel, + applicationTimeouts, tokensConf, applicationTags, null); + } + + public RMApp submitApp(List<ResourceRequest> amResourceRequests, String name, + String user, Map<ApplicationAccessType, String> acls, boolean unmanaged, + String queue, int maxAppAttempts, Credentials ts, String appType, + boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided, + ApplicationId applicationId, long attemptFailuresValidityInterval, + LogAggregationContext logAggregationContext, + boolean cancelTokensWhenComplete, Priority priority, String amLabel, + Map<ApplicationTimeoutType, Long> applicationTimeouts, + ByteBuffer tokensConf, Set<String> applicationTags, String appNodeLabel) + throws Exception { ApplicationId appId = isAppIdProvided ? applicationId : null; ApplicationClientProtocol client = getClientRMService(); if (! isAppIdProvided) { @@ -801,6 +829,9 @@ public class MockRM extends ResourceManager { if (priority != null) { sub.setPriority(priority); } + if (appNodeLabel != null) { + sub.setNodeLabelExpression(appNodeLabel); + } sub.setApplicationType(appType); ContainerLaunchContext clc = Records .newRecord(ContainerLaunchContext.class); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index a8b8d89..d5880ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -88,6 +89,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.api.records.NodeAction; @@ -457,6 +459,49 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { } @Test(timeout = 60000) + public void testAppReportNodeLabelRMRestart() throws Exception { + if (getSchedulerType() != SchedulerType.CAPACITY) { + return; + } + // Create RM + YarnConfiguration newConf = new YarnConfiguration(conf); + newConf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + MockRM rm1 = createMockRM(newConf); + NodeLabel amLabel = NodeLabel.newInstance("AMLABEL"); + NodeLabel appLabel = NodeLabel.newInstance("APPLABEL"); + List<NodeLabel> labels = new ArrayList<>(); + labels.add(amLabel); + labels.add(appLabel); + MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore(); + rm1.start(); + // Add label + rm1.getAdminService().addToClusterNodeLabels( + AddToClusterNodeLabelsRequest.newInstance(labels)); + // create app and launch the AM + ResourceRequest amResourceRequest = ResourceRequest + .newInstance(Priority.newInstance(0), ResourceRequest.ANY, + Resource.newInstance(200, 1), 1, true, amLabel.getName()); + ArrayList resReqs = new ArrayList<>(); + resReqs.add(amResourceRequest); + RMApp app0 = rm1.submitApp(resReqs, appLabel.getName()); + rm1.killApp(app0.getApplicationId()); + rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED); + // start new RM + MockRM rm2 = createMockRM(conf, memStore); + rm2.start(); + Assert.assertEquals(1, rm2.getRMContext().getRMApps().size()); + ApplicationReport appReport = rm2.getClientRMService().getApplicationReport( + GetApplicationReportRequest.newInstance(app0.getApplicationId())) + .getApplicationReport(); + Assert + .assertEquals(amLabel.getName(), appReport.getAmNodeLabelExpression()); + Assert.assertEquals(appLabel.getName(), + appReport.getAppNodeLabelExpression()); + rm1.stop(); + rm2.stop(); + } + + @Test(timeout = 60000) public void testUnManagedRMRestart() throws Exception { // Create RM MockRM rm1 = createMockRM(conf); @@ -471,6 +516,10 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { MockRM rm2 = createMockRM(conf, memStore); rm2.start(); Assert.assertEquals(1, rm2.getRMContext().getRMApps().size()); + ApplicationReport appReport = rm2.getClientRMService().getApplicationReport( + GetApplicationReportRequest.newInstance(app0.getApplicationId())) + .getApplicationReport(); + Assert.assertEquals(true, appReport.isUnmanagedApp()); rm1.stop(); rm2.stop(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
