YARN-8734. Readiness check for remote service belonging to the same user. Contributed by Eric Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1c1dde3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1c1dde3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1c1dde3 Branch: refs/heads/HDFS-13532 Commit: d1c1dde3097eea6a46a39d0312f94949f6884002 Parents: e211350 Author: Billie Rinaldi <bil...@apache.org> Authored: Fri Sep 28 09:39:59 2018 -0700 Committer: Billie Rinaldi <bil...@apache.org> Committed: Fri Sep 28 09:40:48 2018 -0700 ---------------------------------------------------------------------- ...RN-Simplified-V1-API-Layer-For-Services.yaml | 5 +++ .../hadoop/yarn/service/ServiceManager.java | 3 +- .../hadoop/yarn/service/ServiceScheduler.java | 2 + .../yarn/service/api/records/Service.java | 15 +++++++ .../yarn/service/utils/ServiceApiUtil.java | 43 ++++++++++++++++++++ .../yarn/service/TestYarnNativeServices.java | 2 + .../yarn/service/utils/TestServiceApiUtil.java | 39 ++++++++++++++++++ .../markdown/yarn-service/YarnServiceAPI.md | 2 +- 8 files changed, 109 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml index d90ae06..d74604a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml @@ -254,6 +254,11 @@ definitions: docker_client_config: type: string description: URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json). + dependencies: + type: array + items: + type: string + description: An array of services which should be in STABLE state, before this service can be started. ResourceInformation: description: ResourceInformation determines unit/value of resource types in addition to memory and vcores. It will be part of Resource object. http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java index 4851325..aefdadd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java @@ -170,7 +170,8 @@ public class ServiceManager implements EventHandler<ServiceEvent> { } else { serviceManager.setServiceState(ServiceState.UPGRADING); } - + ServiceApiUtil.checkServiceDependencySatisified(serviceManager + .getServiceSpec()); return State.UPGRADING; } catch (Throwable e) { LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java index b49ef2a..9b9305c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java @@ -389,6 +389,8 @@ public class ServiceScheduler extends CompositeService { // Since AM has been started and registered, the service is in STARTED state app.setState(ServiceState.STARTED); + ServiceApiUtil.checkServiceDependencySatisified(context.service); + // recover components based on containers sent from RM recoverComponents(response); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java index 57d1a1a..6e03121 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java @@ -73,6 +73,7 @@ public class Service extends BaseResource { private String version = null; private String description = null; private String dockerClientConfig = null; + private List<String> dependencies = new ArrayList<String>(); /** * A unique service name. @@ -352,6 +353,18 @@ public class Service extends BaseResource { this.queue = queue; } + @ApiModelProperty(example = "null", value = "A list of dependent services.") + @XmlElement(name = "dependencies") + @JsonProperty("dependencies") + public List<String> getDependencies() { + return dependencies; + } + + public void setDependencies(List<String> + dependencies) { + this.dependencies = dependencies; + } + public Service kerberosPrincipal(KerberosPrincipal kerberosPrincipal) { this.kerberosPrincipal = kerberosPrincipal; return this; @@ -437,6 +450,8 @@ public class Service extends BaseResource { .append(toIndentedString(kerberosPrincipal)).append("\n"); sb.append(" dockerClientConfig: ") .append(toIndentedString(dockerClientConfig)).append("\n"); + sb.append(" dependencies: ") + .append(toIndentedString(dependencies)).append("\n"); sb.append("}"); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java index 0eb54ce..d201c7d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java @@ -35,6 +35,8 @@ import org.apache.hadoop.yarn.service.api.records.ComponentState; import org.apache.hadoop.yarn.service.api.records.Container; import org.apache.hadoop.yarn.service.api.records.ContainerState; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; +import org.apache.hadoop.yarn.service.client.ServiceClient; import org.apache.hadoop.yarn.service.api.records.Artifact; import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Configuration; @@ -705,4 +707,45 @@ public class ServiceApiUtil { } return components; } + + private static boolean serviceDependencySatisfied(Service service) { + boolean result = true; + try { + List<String> dependencies = service + .getDependencies(); + org.apache.hadoop.conf.Configuration conf = + new org.apache.hadoop.conf.Configuration(); + if (dependencies != null && dependencies.size() > 0) { + ServiceClient sc = new ServiceClient(); + sc.init(conf); + sc.start(); + for (String dependent : dependencies) { + Service dependentService = sc.getStatus(dependent); + if (dependentService.getState() == null || + !dependentService.getState().equals(ServiceState.STABLE)) { + result = false; + LOG.info("Service dependency is not satisfied for " + + "service: {} state: {}", dependent, + dependentService.getState()); + } + } + sc.close(); + } + } catch (IOException | YarnException e) { + LOG.warn("Caught exception: ", e); + LOG.info("Service dependency is not satisified."); + result = false; + } + return result; + } + + public static void checkServiceDependencySatisified(Service service) { + while (!serviceDependencySatisfied(service)) { + try { + LOG.info("Waiting for service dependencies."); + Thread.sleep(15000L); + } catch (InterruptedException e) { + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java index 3e23a10..d50c574 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java @@ -438,6 +438,8 @@ public class TestYarnNativeServices extends ServiceTestUtils { // wait for upgrade to complete waitForServiceToBeStable(client, service); Service active = client.getStatus(service.getName()); + Assert.assertEquals("version mismatch", service.getVersion(), + active.getVersion()); Assert.assertEquals("component not stable", ComponentState.STABLE, active.getComponent(component.getName()).getState()); Assert.assertEquals("compa does not have new env", "val1", http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java index 98e7474..1e3c180 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestServiceApiUtil.java @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.service.api.records.PlacementScope; import org.apache.hadoop.yarn.service.api.records.PlacementType; import org.apache.hadoop.yarn.service.api.records.Resource; import org.apache.hadoop.yarn.service.api.records.Service; +import org.apache.hadoop.yarn.service.api.records.ServiceState; import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages; import org.junit.Assert; import org.junit.BeforeClass; @@ -733,6 +734,44 @@ public class TestServiceApiUtil extends ServiceTestUtils { } } + @Test(timeout = 1500) + public void testNoServiceDependencies() { + Service service = createExampleApplication(); + Component compa = createComponent("compa"); + Component compb = createComponent("compb"); + service.addComponent(compa); + service.addComponent(compb); + List<String> dependencies = new ArrayList<String>(); + service.setDependencies(dependencies); + ServiceApiUtil.checkServiceDependencySatisified(service); + } + + @Test + public void testServiceDependencies() { + Thread thread = new Thread() { + @Override + public void run() { + Service service = createExampleApplication(); + Component compa = createComponent("compa"); + Component compb = createComponent("compb"); + service.addComponent(compa); + service.addComponent(compb); + List<String> dependencies = new ArrayList<String>(); + dependencies.add("abc"); + service.setDependencies(dependencies); + Service dependent = createExampleApplication(); + dependent.setState(ServiceState.STOPPED); + ServiceApiUtil.checkServiceDependencySatisified(service); + } + }; + thread.start(); + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + } + Assert.assertTrue(thread.isAlive()); + } + public static Service createExampleApplication() { Service exampleApp = new Service(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1c1dde3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md index 4bfa742..7b1e74a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md @@ -406,7 +406,7 @@ a service resource has the following attributes. |queue|The YARN queue that this service should be submitted to.|false|string|| |kerberos_principal|The principal info of the user who launches the service|false|KerberosPrincipal|| |docker_client_config|URI of the file containing the docker client configuration (e.g. hdfs:///tmp/config.json)|false|string|| - +|dependencies|A list of service names that this service depends on.| false | string array || ### ServiceState --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
