hadoop git commit: YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi
Repository: hadoop
Updated Branches:
refs/heads/branch-3.1 6ee62e6b1 -> 148322ca7
YARN-8142. Improve SIGTERM handling for YARN Service Application Master.
Contributed by Billie Rinaldi
(cherry picked from commit 9031a76d447f0c5eaa392144fd17c5b9812e1b20)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/148322ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/148322ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/148322ca
Branch: refs/heads/branch-3.1
Commit: 148322ca72ee7bc40ece551b485f9e939fe4a893
Parents: 6ee62e6
Author: Eric Yang
Authored: Fri Apr 13 15:34:33 2018 -0400
Committer: Billie Rinaldi
Committed: Sat Apr 14 12:10:37 2018 -0700
--
.../hadoop/yarn/service/ClientAMService.java| 1 +
.../hadoop/yarn/service/ServiceScheduler.java | 41 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 11 +++
.../yarn/service/TestYarnNativeServices.java| 71
4 files changed, 110 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/148322ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 08c36f4..3d037e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -125,6 +125,7 @@ public class ClientAMService extends AbstractService
LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser());
context.scheduler.getDiagnostics()
.append("Stopped by user " + UserGroupInformation.getCurrentUser());
+context.scheduler.setGracefulStop();
// Stop the service in 2 seconds delay to make sure this rpc call is
completed.
// shutdown hook will be executed which will stop AM gracefully.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/148322ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 0fcca16..7eddef9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -156,6 +156,8 @@ public class ServiceScheduler extends CompositeService {
// requests for a single service is not recommended.
private boolean hasAtLeastOnePlacementConstraint;
+ private boolean gracefulStop = false;
+
public ServiceScheduler(ServiceContext context) {
super(context.service.getName());
this.context = context;
@@ -199,6 +201,7 @@ public class ServiceScheduler extends CompositeService {
addIfService(amRMClient);
nmClient = createNMClient();
+nmClient.getClient().cleanupRunningContainersOnStop(false);
addIfService(nmClient);
dispatcher = new AsyncDispatcher("Component dispatcher");
@@ -252,6 +255,11 @@ public class ServiceScheduler extends CompositeService {
.createAMRMClientAsync(1000, new AMRMClientCallback());
}
+ protected void setGracefulStop() {
+this.gracefulStop = true;
+nmClient.getClient().cleanupRunningContainersOnStop(true);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
try {
@@ -266,26 +274,31 @@ public class ServiceScheduler extends CompositeService {
public void serviceStop() throws Exception {
LOG.info("Stopping service scheduler");
-// Mar
[42/44] hadoop git commit: YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi
YARN-8142. Improve SIGTERM handling for YARN Service Application Master.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9031a76d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9031a76d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9031a76d
Branch: refs/heads/HDFS-7240
Commit: 9031a76d447f0c5eaa392144fd17c5b9812e1b20
Parents: e66e287
Author: Eric Yang
Authored: Fri Apr 13 15:34:33 2018 -0400
Committer: Eric Yang
Committed: Fri Apr 13 15:34:33 2018 -0400
--
.../hadoop/yarn/service/ClientAMService.java| 1 +
.../hadoop/yarn/service/ServiceScheduler.java | 41 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 11 +++
.../yarn/service/TestYarnNativeServices.java| 71
4 files changed, 110 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 08c36f4..3d037e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -125,6 +125,7 @@ public class ClientAMService extends AbstractService
LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser());
context.scheduler.getDiagnostics()
.append("Stopped by user " + UserGroupInformation.getCurrentUser());
+context.scheduler.setGracefulStop();
// Stop the service in 2 seconds delay to make sure this rpc call is
completed.
// shutdown hook will be executed which will stop AM gracefully.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 0fcca16..7eddef9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -156,6 +156,8 @@ public class ServiceScheduler extends CompositeService {
// requests for a single service is not recommended.
private boolean hasAtLeastOnePlacementConstraint;
+ private boolean gracefulStop = false;
+
public ServiceScheduler(ServiceContext context) {
super(context.service.getName());
this.context = context;
@@ -199,6 +201,7 @@ public class ServiceScheduler extends CompositeService {
addIfService(amRMClient);
nmClient = createNMClient();
+nmClient.getClient().cleanupRunningContainersOnStop(false);
addIfService(nmClient);
dispatcher = new AsyncDispatcher("Component dispatcher");
@@ -252,6 +255,11 @@ public class ServiceScheduler extends CompositeService {
.createAMRMClientAsync(1000, new AMRMClientCallback());
}
+ protected void setGracefulStop() {
+this.gracefulStop = true;
+nmClient.getClient().cleanupRunningContainersOnStop(true);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
try {
@@ -266,26 +274,31 @@ public class ServiceScheduler extends CompositeService {
public void serviceStop() throws Exception {
LOG.info("Stopping service scheduler");
-// Mark component-instances/containers as STOPPED
-if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
- for (ContainerId containerId : getLiveInsta
hadoop git commit: YARN-8142. Improve SIGTERM handling for YARN Service Application Master. Contributed by Billie Rinaldi
Repository: hadoop
Updated Branches:
refs/heads/trunk e66e287ef -> 9031a76d4
YARN-8142. Improve SIGTERM handling for YARN Service Application Master.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9031a76d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9031a76d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9031a76d
Branch: refs/heads/trunk
Commit: 9031a76d447f0c5eaa392144fd17c5b9812e1b20
Parents: e66e287
Author: Eric Yang
Authored: Fri Apr 13 15:34:33 2018 -0400
Committer: Eric Yang
Committed: Fri Apr 13 15:34:33 2018 -0400
--
.../hadoop/yarn/service/ClientAMService.java| 1 +
.../hadoop/yarn/service/ServiceScheduler.java | 41 +++
.../hadoop/yarn/service/ServiceTestUtils.java | 11 +++
.../yarn/service/TestYarnNativeServices.java| 71
4 files changed, 110 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 08c36f4..3d037e7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -125,6 +125,7 @@ public class ClientAMService extends AbstractService
LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser());
context.scheduler.getDiagnostics()
.append("Stopped by user " + UserGroupInformation.getCurrentUser());
+context.scheduler.setGracefulStop();
// Stop the service in 2 seconds delay to make sure this rpc call is
completed.
// shutdown hook will be executed which will stop AM gracefully.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9031a76d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 0fcca16..7eddef9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -156,6 +156,8 @@ public class ServiceScheduler extends CompositeService {
// requests for a single service is not recommended.
private boolean hasAtLeastOnePlacementConstraint;
+ private boolean gracefulStop = false;
+
public ServiceScheduler(ServiceContext context) {
super(context.service.getName());
this.context = context;
@@ -199,6 +201,7 @@ public class ServiceScheduler extends CompositeService {
addIfService(amRMClient);
nmClient = createNMClient();
+nmClient.getClient().cleanupRunningContainersOnStop(false);
addIfService(nmClient);
dispatcher = new AsyncDispatcher("Component dispatcher");
@@ -252,6 +255,11 @@ public class ServiceScheduler extends CompositeService {
.createAMRMClientAsync(1000, new AMRMClientCallback());
}
+ protected void setGracefulStop() {
+this.gracefulStop = true;
+nmClient.getClient().cleanupRunningContainersOnStop(true);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
try {
@@ -266,26 +274,31 @@ public class ServiceScheduler extends CompositeService {
public void serviceStop() throws Exception {
LOG.info("Stopping service scheduler");
-// Mark component-instances/containers as STOPPED
-if (YarnConfiguration.timelineServic
