YARN-8411. Restart stopped system service during RM start.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69b05968
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69b05968
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69b05968
Branch: refs/heads/HDFS-13532
Commit: 69b05968974994c6e22d6562a67b9392d1700094
Parents: 7566e0e
Author: Eric Yang <[email protected]>
Authored: Wed Jun 13 19:05:52 2018 -0400
Committer: Eric Yang <[email protected]>
Committed: Wed Jun 13 19:05:52 2018 -0400
----------------------------------------------------------------------
.../client/SystemServiceManagerImpl.java | 29 ++++++++++++--
.../hadoop/yarn/service/webapp/ApiServer.java | 3 +-
.../hadoop/yarn/service/ServiceClientTest.java | 4 +-
.../client/TestSystemServiceManagerImpl.java | 40 +++++++++++++++++---
.../yarn/service/client/ServiceClient.java | 14 +++++--
5 files changed, 72 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b05968/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
index f9cfa92..08ad1b6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/SystemServiceManagerImpl.java
@@ -29,7 +29,9 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.service.SystemServiceManager;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -228,12 +230,31 @@ public class SystemServiceManagerImpl extends
AbstractService
userUgi.doAs(new PrivilegedExceptionAction<ApplicationId>() {
@Override public ApplicationId run()
throws IOException, YarnException {
- ApplicationId applicationId =
serviceClient.actionCreate(service);
- return applicationId;
+ boolean tryStart = true;
+ try {
+ serviceClient.actionBuild(service);
+ } catch (Exception e) {
+ if (e instanceof SliderException && ((SliderException) e)
+ .getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+ LOG.info("Service {} already exists, will attempt to start "
+
+ "service", service.getName());
+ } else {
+ tryStart = false;
+ LOG.info("Got exception saving {}, will not attempt to " +
+ "start service", service.getName(), e);
+ }
+ }
+ if (tryStart) {
+ return serviceClient.actionStartAndGetId(service.getName());
+ } else {
+ return null;
+ }
}
});
- LOG.info("Service {} submitted with Application ID: {}",
- service.getName(), applicationId);
+ if (applicationId != null) {
+ LOG.info("Service {} submitted with Application ID: {}",
+ service.getName(), applicationId);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b05968/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 578273c..82fadae 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -648,8 +648,7 @@ public class ApiServer {
ServiceClient sc = getServiceClient();
sc.init(YARN_CONFIG);
sc.start();
- sc.actionStart(appName);
- ApplicationId appId = sc.getAppId(appName);
+ ApplicationId appId = sc.actionStartAndGetId(appName);
sc.close();
return appId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b05968/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
index 81be750..d022614 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/ServiceClientTest.java
@@ -103,13 +103,13 @@ public class ServiceClientTest extends ServiceClient {
}
@Override
- public int actionStart(String serviceName)
+ public ApplicationId actionStartAndGetId(String serviceName)
throws YarnException, IOException {
if (serviceName != null && serviceName.equals("jenkins")) {
ApplicationId appId =
ApplicationId.newInstance(System.currentTimeMillis(), 1);
serviceAppId.put(serviceName, appId);
- return EXIT_SUCCESS;
+ return appId;
} else {
throw new ApplicationNotFoundException("");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b05968/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
index d39083d..892f2b4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/client/TestSystemServiceManagerImpl.java
@@ -22,7 +22,9 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -51,6 +53,7 @@ public class TestSystemServiceManagerImpl {
private String[] users = new String[] {"user1", "user2"};
private static Map<String, Set<String>> loadedServices = new HashMap<>();
+ private static Map<String, Set<String>> savedServices = new HashMap<>();
private static Map<String, Set<String>> submittedServices = new HashMap<>();
@Before
@@ -72,7 +75,7 @@ public class TestSystemServiceManagerImpl {
}
@After
- public void teadDown() {
+ public void tearDown() {
systemService.stop();
}
@@ -102,6 +105,11 @@ public class TestSystemServiceManagerImpl {
// 2nd time launch service to handle if service exist scenario
systemService.launchUserService(userServices);
verifyForLaunchedUserServices();
+
+ // verify start of stopped services
+ submittedServices.clear();
+ systemService.launchUserService(userServices);
+ verifyForLaunchedUserServices();
}
private void verifyForScannedUserServices(
@@ -149,21 +157,41 @@ public class TestSystemServiceManagerImpl {
}
@Override
- public ApplicationId actionCreate(Service service)
+ public int actionBuild(Service service)
throws YarnException, IOException {
String userName =
UserGroupInformation.getCurrentUser().getShortUserName();
- Set<String> services = submittedServices.get(userName);
+ Set<String> services = savedServices.get(userName);
if (services == null) {
services = new HashSet<>();
- submittedServices.put(userName, services);
+ savedServices.put(userName, services);
}
if (services.contains(service.getName())) {
- String message = "Failed to create service " + service.getName()
+ String message = "Failed to save service " + service.getName()
+ ", because it already exists.";
- throw new YarnException(message);
+ throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS,
+ message);
}
services.add(service.getName());
+ return 0;
+ }
+
+ @Override
+ public ApplicationId actionStartAndGetId(String serviceName)
+ throws YarnException, IOException {
+ String userName =
+ UserGroupInformation.getCurrentUser().getShortUserName();
+ Set<String> services = submittedServices.get(userName);
+ if (services == null) {
+ services = new HashSet<>();
+ submittedServices.put(userName, services);
+ }
+ if (services.contains(serviceName)) {
+ String message = "Failed to create service " + serviceName
+ + ", because it is already running.";
+ throw new YarnException(message);
+ }
+ services.add(serviceName);
return ApplicationId.newInstance(System.currentTimeMillis(), 1);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b05968/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index b86120f..699a4e5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -1003,6 +1003,12 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
@Override
public int actionStart(String serviceName) throws YarnException, IOException
{
+ actionStartAndGetId(serviceName);
+ return EXIT_SUCCESS;
+ }
+
+ public ApplicationId actionStartAndGetId(String serviceName) throws
+ YarnException, IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
Service liveService = getStatus(serviceName);
if (liveService == null ||
@@ -1019,11 +1025,11 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
// write app definition on to hdfs
Path appJson = ServiceApiUtil.writeAppDefinition(fs, appDir, service);
LOG.info("Persisted service " + service.getName() + " at " + appJson);
- return 0;
+ return appId;
} else {
LOG.info("Finalize service {} upgrade");
- ApplicationReport appReport =
- yarnClient.getApplicationReport(getAppId(serviceName));
+ ApplicationId appId = getAppId(serviceName);
+ ApplicationReport appReport = yarnClient.getApplicationReport(appId);
if (StringUtils.isEmpty(appReport.getHost())) {
throw new YarnException(serviceName + " AM hostname is empty");
}
@@ -1032,7 +1038,7 @@ public class ServiceClient extends AppAdminClient
implements SliderExitCodes,
RestartServiceRequestProto.Builder requestBuilder =
RestartServiceRequestProto.newBuilder();
proxy.restart(requestBuilder.build());
- return 0;
+ return appId;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
