hadoop git commit: YARN-7827. Stop and Delete Yarn Service from RM UI fails with HTTP ERROR 404. Contributed by Sunil G

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 1bc03ddf9 -> ddec08d7c


YARN-7827. Stop and Delete Yarn Service from RM UI fails with HTTP ERROR 404. 
Contributed by Sunil G


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddec08d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddec08d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddec08d7

Branch: refs/heads/trunk
Commit: ddec08d7ccc8e43492fca2784203bd8af5e968cc
Parents: 1bc03dd
Author: Jian He 
Authored: Thu Feb 8 21:32:02 2018 -0800
Committer: Jian He 
Committed: Thu Feb 8 21:32:40 2018 -0800

--
 .../src/main/webapp/app/adapters/yarn-servicedef.js |  9 ++---
 .../src/main/webapp/app/components/deploy-service.js| 12 +---
 .../src/main/webapp/app/controllers/yarn-app.js |  4 ++--
 .../src/main/webapp/app/controllers/yarn-app/info.js|  4 ++--
 .../main/webapp/app/controllers/yarn-deploy-service.js  | 12 ++--
 .../webapp/app/templates/components/deploy-service.hbs  | 10 ++
 .../src/main/webapp/app/templates/yarn-app.hbs  |  4 ++--
 .../src/main/webapp/app/templates/yarn-app/info.hbs |  4 ++--
 .../src/main/webapp/app/utils/info-seeder.js|  3 ++-
 .../src/main/webapp/config/default-config.js|  2 +-
 10 files changed, 42 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddec08d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
index 3fb4a81..03685fb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
@@ -24,21 +24,24 @@ export default RESTAbstractAdapter.extend({
   restNameSpace: "dashService",
   serverName: "DASH",
 
-  deployService(request) {
+  deployService(request, user) {
 var url = this.buildURL();
+url += "/?user.name=" + user;
 return this.ajax(url, "POST", {data: request});
   },
 
-  stopService(serviceName) {
+  stopService(serviceName, user) {
 var url = this.buildURL();
 url += "/" + serviceName;
+url += "/?user.name=" + user;
 var data = {"state": "STOPPED", "name": serviceName};
 return this.ajax(url, "PUT", {data: data});
   },
 
-  deleteService(serviceName) {
+  deleteService(serviceName, user) {
 var url = this.buildURL();
 url += "/" + serviceName;
+url += "/?user.name=" + user;
 return this.ajax(url, "DELETE", {data: {}});
   }
 });

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddec08d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js
index 90e10e5..36895d7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/deploy-service.js
@@ -27,6 +27,7 @@ export default Ember.Component.extend({
   customServiceDef: '',
   serviceResp: null,
   isLoading: false,
+  userName: '',
 
   actions: {
 showSaveTemplateModal() {
@@ -36,11 +37,11 @@ export default Ember.Component.extend({
 deployService() {
   this.set('serviceResp', null);
   if (this.get('isStandardViewType')) {
-this.sendAction("deployServiceDef", this.get('serviceDef'));
+this.sendAction("deployServiceDef", this.get('serviceDef'), 
this.get('userName'));
   } else {
 try {
   var parsed = JSON.parse(this.get('customServiceDef'));
-  this.sendAction("deployServiceJson", parsed);
+  this.sendAction("deployServiceJson", parsed, this.get('userName'));
 } catch (err) {
   this.set('serviceResp', {type: 'error', message: 'Invalid JSON: ' + 
err.message});
   throw err;
@@ -148,16 +149,21 @@ export default Ember.Component.extend({
 
   isValidTemplateName: Ember.computed.notEmpty('savedTemplateName'),
 
+  isUserNameGiven: Ember.computed.empty('userName'),
+
   isValidServiceDef: Ember.computed('serviceDef.name', 'serviceDef.queue', 
'serviceDef.serviceComponents.[]', function () {
 return 

hadoop git commit: YARN-5428. Allow for specifying the docker client configuration directory. Contributed by Shane Kumpf

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 996796f10 -> eb2449d53


YARN-5428. Allow for specifying the docker client configuration directory. 
Contributed by Shane Kumpf


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb2449d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb2449d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb2449d5

Branch: refs/heads/trunk
Commit: eb2449d5398e9ac869bc088e10d838a7f13deac0
Parents: 996796f
Author: Jian He 
Authored: Wed Feb 7 10:59:38 2018 -0800
Committer: Jian He 
Committed: Thu Feb 8 11:35:30 2018 -0800

--
 .../applications/distributedshell/Client.java   |  38 +++-
 .../DockerCredentialTokenIdentifier.java| 159 
 .../yarn/util/DockerClientConfigHandler.java| 183 +++
 .../src/main/proto/yarn_security_token.proto|   5 +
 ...apache.hadoop.security.token.TokenIdentifier |   1 +
 .../security/TestDockerClientConfigHandler.java | 129 +
 .../runtime/DockerLinuxContainerRuntime.java|  39 
 .../linux/runtime/docker/DockerCommand.java |  16 ++
 .../runtime/TestDockerContainerRuntime.java | 109 +++
 .../runtime/docker/TestDockerRunCommand.java|   8 +
 .../src/site/markdown/DockerContainers.md   |  13 +-
 11 files changed, 690 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb2449d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 2aafa94..0aef83f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -87,6 +87,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.DockerClientConfigHandler;
 import org.apache.hadoop.yarn.util.UnitsConversionUtil;
 import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -225,6 +226,9 @@ public class Client {
   private String flowVersion = null;
   private long flowRunId = 0L;
 
+  // Docker client configuration
+  private String dockerClientConfig = null;
+
   // Command line options
   private Options opts;
 
@@ -368,6 +372,10 @@ public class Client {
 "If container could retry, it specifies max retires");
 opts.addOption("container_retry_interval", true,
 "Interval between each retry, unit is milliseconds");
+opts.addOption("docker_client_config", true,
+"The docker client configuration path. The scheme should be supplied"
++ " (i.e. file:// or hdfs://)."
++ " Only used when the Docker runtime is enabled and requested.");
 opts.addOption("placement_spec", true,
 "Placement specification. Please note, if this option is specified,"
 + " The \"num_containers\" option will be ignored. All requested"
@@ -585,6 +593,9 @@ public class Client {
 "Flow run is not a valid long value", e);
   }
 }
+if (cliParser.hasOption("docker_client_config")) {
+  dockerClientConfig = cliParser.getOptionValue("docker_client_config");
+}
 return true;
   }
 
@@ -884,9 +895,10 @@ public class Client {
 // amContainer.setServiceData(serviceData);
 
 // Setup security tokens
+Credentials rmCredentials = null;
 if (UserGroupInformation.isSecurityEnabled()) {
   // Note: Credentials class is marked as LimitedPrivate for HDFS and 
MapReduce
-  Credentials credentials = new Credentials();
+  rmCredentials = new Credentials();
   String tokenRenewer = YarnClientUtils.getRmPrincipal(conf);
   if (tokenRenewer == null || tokenRenewer.length() == 0) {
 throw new IOException(
@@ -895,16 +907,32 @@ public class Client {
 
   // 

hadoop git commit: YARN-7889. Missing kerberos token when check for RM REST API availability. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a196ee936 -> 87429affc


YARN-7889. Missing kerberos token when check for RM REST API availability. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87429aff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87429aff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87429aff

Branch: refs/heads/trunk
Commit: 87429affcea2de61791c1b048b6db8c8275de051
Parents: a196ee9
Author: Jian He 
Authored: Mon Feb 5 22:55:56 2018 -0800
Committer: Jian He 
Committed: Mon Feb 5 22:56:25 2018 -0800

--
 .../yarn/service/client/ApiServiceClient.java   | 41 ++--
 1 file changed, 29 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87429aff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index cb91946..49702e3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -87,21 +87,38 @@ public class ApiServiceClient extends AppAdminClient {
   rmAddress = conf
   .get("yarn.resourcemanager.webapp.https.address");
 }
-
+boolean useKerberos = UserGroupInformation.isSecurityEnabled();
 List rmServers = RMHAUtils
 .getRMHAWebappAddresses(new YarnConfiguration(conf));
 for (String host : rmServers) {
-  StringBuilder sb = new StringBuilder();
-  sb.append(scheme);
-  sb.append(host);
-  sb.append(path);
-  Client client = Client.create();
-  WebResource webResource = client
-  .resource(sb.toString());
-  String test = webResource.get(String.class);
-  if (test.contains("hadoop_version")) {
-rmAddress = host;
-break;
+  try {
+Client client = Client.create();
+StringBuilder sb = new StringBuilder();
+sb.append(scheme);
+sb.append(host);
+sb.append(path);
+if (!useKerberos) {
+  try {
+String username = 
UserGroupInformation.getCurrentUser().getShortUserName();
+sb.append("?user.name=");
+sb.append(username);
+  } catch (IOException e) {
+LOG.debug("Fail to resolve username: {}", e);
+  }
+}
+WebResource webResource = client
+.resource(sb.toString());
+if (useKerberos) {
+  AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+  webResource.header("WWW-Authenticate", token);
+}
+ClientResponse test = webResource.get(ClientResponse.class);
+if (test.getStatus() == 200) {
+  rmAddress = host;
+  break;
+}
+  } catch (Exception e) {
+LOG.debug("Fail to connect to: "+host, e);
   }
 }
 return scheme+rmAddress;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7831. YARN Service CLI should use hadoop.http.authentication.type to determine authentication method. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 51cb6c538 -> b6e50fad5


YARN-7831. YARN Service CLI should use hadoop.http.authentication.type to 
determine authentication method. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6e50fad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6e50fad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6e50fad

Branch: refs/heads/trunk
Commit: b6e50fad53f26e2b718a85ec0678e3161decc691
Parents: 51cb6c5
Author: Jian He 
Authored: Fri Feb 2 14:37:58 2018 -0800
Committer: Jian He 
Committed: Fri Feb 2 14:38:33 2018 -0800

--
 .../org/apache/hadoop/yarn/service/client/ApiServiceClient.java | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6e50fad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 8c2edb5..cb91946 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -123,7 +123,8 @@ public class ApiServiceClient extends AppAdminClient {
   api.append("/");
   api.append(appName);
 }
-if (!UserGroupInformation.isSecurityEnabled()) {
+Configuration conf = getConfig();
+if 
(conf.get("hadoop.http.authentication.type").equalsIgnoreCase("simple")) {
   api.append("?user.name=" + UrlEncoded
   .encodeString(System.getProperty("user.name")));
 }
@@ -147,7 +148,7 @@ public class ApiServiceClient extends AppAdminClient {
 client.setChunkedEncodingSize(null);
 Builder builder = client
 .resource(getApiUrl(appName)).type(MediaType.APPLICATION_JSON);
-if (conf.get("hadoop.security.authentication").equals("kerberos")) {
+if (conf.get("hadoop.http.authentication.type").equals("kerberos")) {
   AuthenticatedURL.Token token = new AuthenticatedURL.Token();
   builder.header("WWW-Authenticate", token);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7868. Provide improved error message when YARN service is disabled. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 6e5ba9366 -> f9a4d4cf2


YARN-7868. Provide improved error message when YARN service is disabled. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9a4d4cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9a4d4cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9a4d4cf

Branch: refs/heads/trunk
Commit: f9a4d4cf237d64ccb52ab1792372c89e9d36f41d
Parents: 6e5ba93
Author: Jian He 
Authored: Fri Feb 2 11:10:18 2018 -0800
Committer: Jian He 
Committed: Fri Feb 2 11:10:47 2018 -0800

--
 .../org/apache/hadoop/yarn/service/client/ApiServiceClient.java  | 4 
 1 file changed, 4 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a4d4cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 34e62b6..8c2edb5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -171,6 +171,10 @@ public class ApiServiceClient extends AppAdminClient {
   LOG.error("Authentication required");
   return EXIT_EXCEPTION_THROWN;
 }
+if (response.getStatus() == 503) {
+  LOG.error("YARN Service is unavailable or disabled.");
+  return EXIT_EXCEPTION_THROWN;
+}
 try {
   ServiceStatus ss = response.getEntity(ServiceStatus.class);
   output = ss.getDiagnostics();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 312c57169 -> 0497e0933


YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline 
events. Contributed by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0497e093
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0497e093
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0497e093

Branch: refs/heads/branch-2
Commit: 0497e09339415ffd6b6f84beeff7e1809647662c
Parents: 312c571
Author: Jian He 
Authored: Sat Jan 27 22:18:51 2018 -0800
Committer: Jian He 
Committed: Sun Jan 28 13:24:49 2018 -0800

--
 .../hadoop/yarn/server/nodemanager/NodeManager.java  | 11 ---
 .../nodemanager/timelineservice/NMTimelinePublisher.java |  1 +
 .../timelineservice/storage/HBaseTimelineWriterImpl.java |  5 +
 3 files changed, 10 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0497e093/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index c23c071..431a894 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -440,18 +440,15 @@ public class NodeManager extends CompositeService
 addService(nodeStatusUpdater);
 ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
 
-super.serviceInit(conf);
-// TODO add local dirs to del
-  }
-
-  @Override
-  protected void serviceStart() throws Exception {
+// Do secure login before calling init for added services.
 try {
   doSecureLogin();
 } catch (IOException e) {
   throw new YarnRuntimeException("Failed NodeManager login", e);
 }
-super.serviceStart();
+
+super.serviceInit(conf);
+// TODO add local dirs to del
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0497e093/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 2124c1a..52895c4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -99,6 +99,7 @@ public class NMTimelinePublisher extends CompositeService {
 this.nmLoginUGI =  UserGroupInformation.isSecurityEnabled() ?
 UserGroupInformation.getLoginUser() :
 UserGroupInformation.getCurrentUser();
+LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI);
 super.serviceInit(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0497e093/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 9e9134c..f938185 100644
--- 

hadoop git commit: YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 ec4f5f0f0 -> 331677936


YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline 
events. Contributed by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33167793
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33167793
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33167793

Branch: refs/heads/branch-3.0
Commit: 33167793641857a1be3032ff5b8288f124bf92f7
Parents: ec4f5f0
Author: Jian He 
Authored: Sat Jan 27 22:18:51 2018 -0800
Committer: Jian He 
Committed: Sat Jan 27 22:20:49 2018 -0800

--
 .../hadoop/yarn/server/nodemanager/NodeManager.java  | 11 ---
 .../nodemanager/timelineservice/NMTimelinePublisher.java |  1 +
 .../timelineservice/storage/HBaseTimelineWriterImpl.java |  5 +
 3 files changed, 10 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/33167793/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index a97b3f2..fc48450 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -437,18 +437,15 @@ public class NodeManager extends CompositeService
 addService(nodeStatusUpdater);
 ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
 
-super.serviceInit(conf);
-// TODO add local dirs to del
-  }
-
-  @Override
-  protected void serviceStart() throws Exception {
+// Do secure login before calling init for added services.
 try {
   doSecureLogin();
 } catch (IOException e) {
   throw new YarnRuntimeException("Failed NodeManager login", e);
 }
-super.serviceStart();
+
+super.serviceInit(conf);
+// TODO add local dirs to del
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/33167793/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index b8192ca..2ce3c5e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -100,6 +100,7 @@ public class NMTimelinePublisher extends CompositeService {
 this.nmLoginUGI =  UserGroupInformation.isSecurityEnabled() ?
 UserGroupInformation.getLoginUser() :
 UserGroupInformation.getCurrentUser();
+LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI);
 super.serviceInit(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/33167793/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 9e9134c..f938185 100644
--- 

hadoop git commit: YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline events. Contributed by Rohith Sharma K S

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 649ef7ac3 -> 443523f9c


YARN-7765. Fixed an issue that kerberos tgt not found when NM posting timeline 
events. Contributed by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/443523f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/443523f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/443523f9

Branch: refs/heads/trunk
Commit: 443523f9c0fcc4ba2503791090b1879c6031759b
Parents: 649ef7a
Author: Jian He 
Authored: Sat Jan 27 22:18:51 2018 -0800
Committer: Jian He 
Committed: Sat Jan 27 22:18:51 2018 -0800

--
 .../hadoop/yarn/server/nodemanager/NodeManager.java  | 11 ---
 .../nodemanager/timelineservice/NMTimelinePublisher.java |  1 +
 .../timelineservice/storage/HBaseTimelineWriterImpl.java |  5 +
 3 files changed, 10 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 5cacd20..42b7b5f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -454,18 +454,15 @@ public class NodeManager extends CompositeService
 ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
 nmStore.setNodeStatusUpdater(nodeStatusUpdater);
 
-super.serviceInit(conf);
-// TODO add local dirs to del
-  }
-
-  @Override
-  protected void serviceStart() throws Exception {
+// Do secure login before calling init for added services.
 try {
   doSecureLogin();
 } catch (IOException e) {
   throw new YarnRuntimeException("Failed NodeManager login", e);
 }
-super.serviceStart();
+
+super.serviceInit(conf);
+// TODO add local dirs to del
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index b8192ca..2ce3c5e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -100,6 +100,7 @@ public class NMTimelinePublisher extends CompositeService {
 this.nmLoginUGI =  UserGroupInformation.isSecurityEnabled() ?
 UserGroupInformation.getLoginUser() :
 UserGroupInformation.getCurrentUser();
+LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI);
 super.serviceInit(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/443523f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 9e9134c..f938185 100644
--- 

hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 258fb117d -> cbaf6d242


YARN-7801. AmFilterInitializer should addFilter after fill all parameters. 
Contributed by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbaf6d24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbaf6d24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbaf6d24

Branch: refs/heads/branch-2.9
Commit: cbaf6d242d5b8c878cfd23f4dd4c438a40e967b4
Parents: 258fb11
Author: Jian He 
Authored: Wed Jan 24 11:16:54 2018 -0800
Committer: Jian He 
Committed: Wed Jan 24 16:18:36 2018 -0800

--
 .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbaf6d24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index c3ddc54..27742e4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer {
 }
 sb.setLength(sb.length() - 1);
 params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
-container.addFilter(FILTER_NAME, FILTER_CLASS, params);
 
 // Handle RM HA urls
 // Include yarn-site.xml in the classpath
@@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer {
 params.put(RM_HA_URLS, StringUtils.join(",", urls));
   }
 }
+
+container.addFilter(FILTER_NAME, FILTER_CLASS, params);
   }
 
   private Collection getRmIds(Configuration conf) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 ecfdd01be -> 7598f7332


YARN-7801. AmFilterInitializer should addFilter after fill all parameters. 
Contributed by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7598f733
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7598f733
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7598f733

Branch: refs/heads/branch-3.0
Commit: 7598f7332042f5c2733e191567f1b1420f1e390a
Parents: ecfdd01
Author: Jian He 
Authored: Wed Jan 24 11:16:54 2018 -0800
Committer: Jian He 
Committed: Wed Jan 24 11:20:23 2018 -0800

--
 .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7598f733/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index c3ddc54..27742e4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer {
 }
 sb.setLength(sb.length() - 1);
 params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
-container.addFilter(FILTER_NAME, FILTER_CLASS, params);
 
 // Handle RM HA urls
 // Include yarn-site.xml in the classpath
@@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer {
 params.put(RM_HA_URLS, StringUtils.join(",", urls));
   }
 }
+
+container.addFilter(FILTER_NAME, FILTER_CLASS, params);
   }
 
   private Collection getRmIds(Configuration conf) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ce43e93f9 -> c611707e2


YARN-7801. AmFilterInitializer should addFilter after fill all parameters. 
Contributed by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c611707e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c611707e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c611707e

Branch: refs/heads/branch-2
Commit: c611707e28c7094ff0c8a9558db1c7d04bbc8b6d
Parents: ce43e93
Author: Jian He 
Authored: Wed Jan 24 11:16:54 2018 -0800
Committer: Jian He 
Committed: Wed Jan 24 11:19:46 2018 -0800

--
 .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c611707e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index c3ddc54..27742e4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer {
 }
 sb.setLength(sb.length() - 1);
 params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
-container.addFilter(FILTER_NAME, FILTER_CLASS, params);
 
 // Handle RM HA urls
 // Include yarn-site.xml in the classpath
@@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer {
 params.put(RM_HA_URLS, StringUtils.join(",", urls));
   }
 }
+
+container.addFilter(FILTER_NAME, FILTER_CLASS, params);
   }
 
   private Collection getRmIds(Configuration conf) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7801. AmFilterInitializer should addFilter after fill all parameters. Contributed by Wangda Tan

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 97607617c -> 55c32776b


YARN-7801. AmFilterInitializer should addFilter after fill all parameters. 
Contributed by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55c32776
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55c32776
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55c32776

Branch: refs/heads/trunk
Commit: 55c32776b1f130c6b81ef7aff5fc1562180f4601
Parents: 9760761
Author: Jian He 
Authored: Wed Jan 24 11:16:54 2018 -0800
Committer: Jian He 
Committed: Wed Jan 24 11:17:12 2018 -0800

--
 .../hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/55c32776/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
index c3ddc54..27742e4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java
@@ -59,7 +59,6 @@ public class AmFilterInitializer extends FilterInitializer {
 }
 sb.setLength(sb.length() - 1);
 params.put(AmIpFilter.PROXY_URI_BASES, sb.toString());
-container.addFilter(FILTER_NAME, FILTER_CLASS, params);
 
 // Handle RM HA urls
 // Include yarn-site.xml in the classpath
@@ -75,6 +74,8 @@ public class AmFilterInitializer extends FilterInitializer {
 params.put(RM_HA_URLS, StringUtils.join(",", urls));
   }
 }
+
+container.addFilter(FILTER_NAME, FILTER_CLASS, params);
   }
 
   private Collection getRmIds(Configuration conf) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ab482ebe7 -> a80c2dbab


YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a80c2dba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a80c2dba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a80c2dba

Branch: refs/heads/branch-2
Commit: a80c2dbab5ddfe71aa66f529629fa9964c7689b6
Parents: ab482eb
Author: Jian He 
Authored: Tue Jan 23 14:03:53 2018 -0800
Committer: Jian He 
Committed: Tue Jan 23 14:12:00 2018 -0800

--
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a80c2dba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 905f4da..a975a5b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1667,7 +1667,7 @@
   
 Property to enable docker user remapping
 
yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed
-false
+true
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.9 93220f90e -> 6da08837d


YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6da08837
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6da08837
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6da08837

Branch: refs/heads/branch-2.9
Commit: 6da08837da9bb0f409626b41c9583d6d15da9a5b
Parents: 93220f9
Author: Jian He 
Authored: Tue Jan 23 14:03:53 2018 -0800
Committer: Jian He 
Committed: Tue Jan 23 14:07:41 2018 -0800

--
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da08837/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 71dd72a..ce8c0f1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1659,7 +1659,7 @@
   
 Property to enable docker user remapping
 
yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed
-false
+true
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 1de071257 -> 92f789a13


YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92f789a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92f789a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92f789a1

Branch: refs/heads/branch-3.0
Commit: 92f789a131d8ae5ee46302f3c88554e7314f516f
Parents: 1de0712
Author: Jian He 
Authored: Tue Jan 23 14:03:53 2018 -0800
Committer: Jian He 
Committed: Tue Jan 23 14:06:48 2018 -0800

--
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92f789a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4e26be6..f3a9b09 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1656,7 +1656,7 @@
   
 Property to enable docker user remapping
 
yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed
-false
+true
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a72cdcc47 -> 39b999aba


YARN-7782. Enable user re-mapping for Docker containers in yarn-default.xml. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39b999ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39b999ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39b999ab

Branch: refs/heads/trunk
Commit: 39b999aba2c1a1fc1df17bdf952834ba516f3b2a
Parents: a72cdcc
Author: Jian He 
Authored: Tue Jan 23 14:03:53 2018 -0800
Committer: Jian He 
Committed: Tue Jan 23 14:03:53 2018 -0800

--
 .../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/39b999ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a9938c3..45aa635 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1688,7 +1688,7 @@
   
 Property to enable docker user remapping
 
yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed
-false
+true
   
 
   


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk f63d13f10 -> a72cdcc47


YARN-7766. Introduce a new config property for YARN Service dependency tarball 
location. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a72cdcc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a72cdcc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a72cdcc4

Branch: refs/heads/trunk
Commit: a72cdcc47a1b3a4fcadbc9e12bd1bae2bbf04d83
Parents: f63d13f
Author: Jian He 
Authored: Tue Jan 23 10:53:27 2018 -0800
Committer: Jian He 
Committed: Tue Jan 23 10:53:27 2018 -0800

--
 .../yarn/service/client/ServiceClient.java  | 35 +++-
 .../yarn/service/conf/YarnServiceConf.java  |  6 
 .../tarball/TarballProviderService.java |  4 +--
 .../yarn/service/utils/CoreFileSystem.java  | 32 --
 .../hadoop/yarn/client/api/AppAdminClient.java  | 12 ---
 .../hadoop/yarn/client/cli/ApplicationCLI.java  | 13 +---
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  5 ++-
 7 files changed, 68 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index c224089..b8034b3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.*;
 import org.apache.hadoop.yarn.client.api.AppAdminClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
 import org.apache.hadoop.yarn.client.util.YarnClientUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -718,14 +719,20 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 libPath, "lib", false);
 Path dependencyLibTarGzip = fs.getDependencyTarGzip();
 if (fs.isFile(dependencyLibTarGzip)) {
-  LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
-  + dependencyLibTarGzip);
+  LOG.info("Loading lib tar from " + dependencyLibTarGzip);
   fs.submitTarGzipAndUpdate(localResources);
 } else {
+  if (dependencyLibTarGzip != null) {
+LOG.warn("Property {} has a value {}, but is not a valid file",
+YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
+  }
   String[] libs = ServiceUtils.getLibDirs();
-  LOG.info("Uploading all dependency jars to HDFS. For faster submission 
of" +
-  " apps, pre-upload dependency jars to HDFS "
-  + "using command: yarn app -enableFastLaunch");
+  LOG.info("Uploading all dependency jars to HDFS. For faster submission 
of"
+  + " apps, set config property {} to the dependency tarball location."
+  + " Dependency tarball can be uploaded to any HDFS path directly"
+  + " or by using command: yarn app -{} []",
+  YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+  ApplicationCLI.ENABLE_FAST_LAUNCH);
   for (String libDirProp : libs) {
 ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
 libDirProp);
@@ -988,16 +995,23 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 return this.yarnClient;
   }
 
-  public int enableFastLaunch() throws IOException, YarnException {
-return actionDependency(true);
+  public int enableFastLaunch(String destinationFolder)
+  throws IOException, YarnException {
+return actionDependency(destinationFolder, true);
   }
 
-  public int actionDependency(boolean overwrite)
+  public int actionDependency(String destinationFolder, boolean overwrite)
   throws IOException, YarnException {
 String currentUser = 

hadoop git commit: YARN-7671. Improve Diagonstic message for stop yarn native service. Contributed by Chandni Singh

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 3d65dbe03 -> c05b84703


YARN-7671. Improve Diagonstic message for stop yarn native service. Contributed 
by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c05b8470
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c05b8470
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c05b8470

Branch: refs/heads/trunk
Commit: c05b84703b7754b6c2fbcec39f22b5d7802563ec
Parents: 3d65dbe
Author: Jian He 
Authored: Fri Jan 12 14:04:15 2018 -0800
Committer: Jian He 
Committed: Fri Jan 12 14:04:15 2018 -0800

--
 .../org/apache/hadoop/yarn/service/ClientAMService.java  |  6 +-
 .../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 ---
 2 files changed, 9 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b8470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 94dd8d6..fb73f15 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -117,7 +118,10 @@ public class ClientAMService extends AbstractService
   @Override
   public StopResponseProto stop(StopRequestProto requestProto)
   throws IOException, YarnException {
-LOG.info("Stop the service.");
+LOG.info("Stop the service by {}", UserGroupInformation.getCurrentUser());
+context.scheduler.getDiagnostics()
+.append("Stopped by user " + UserGroupInformation.getCurrentUser());
+
 // Stop the service in 2 seconds delay to make sure this rpc call is 
completed.
 // shutdown hook will be executed which will stop AM gracefully.
 Thread thread = new Thread() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c05b8470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 0a4ea07..eb4783f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -262,13 +262,10 @@ public class ServiceScheduler extends CompositeService {
   serviceTimelinePublisher
   .serviceAttemptUnregistered(context, diagnostics.toString());
 }
-String msg = diagnostics.toString()
-+ "Navigate to the failed component for more details.";
-amRMClient
-.unregisterApplicationMaster(FinalApplicationStatus.ENDED, msg, "");
-LOG.info("Service " + app.getName()
-+ " unregistered with RM, with attemptId = " + context.attemptId
-+ ", diagnostics = " + diagnostics);
+amRMClient.unregisterApplicationMaster(FinalApplicationStatus.ENDED,
+diagnostics.toString(), "");
+LOG.info("Service {} unregistered with RM, with attemptId = {} " +
+", diagnostics = {} ", app.getName(), context.attemptId, diagnostics);
 super.serviceStop();
   

hadoop git commit: Revert "YARN-7540. Route YARN service CLI function through YARN Service API. (Contributed by Eric Yang)"

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a81144daa -> 836e3c45e


Revert "YARN-7540.  Route YARN service CLI function through YARN Service API. 
(Contributed by Eric Yang)"

This reverts commit 438c1d333ebc0a3071bb556532ed959a4bd1e6d6.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/836e3c45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/836e3c45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/836e3c45

Branch: refs/heads/trunk
Commit: 836e3c45e8232fc4c8c795c0f93d2f3d7353f392
Parents: a81144d
Author: Jian He 
Authored: Fri Jan 5 18:25:00 2018 -0800
Committer: Jian He 
Committed: Fri Jan 5 18:25:00 2018 -0800

--
 .../hadoop-yarn-services-api/pom.xml|   9 -
 .../yarn/service/client/ApiServiceClient.java   | 430 ---
 .../hadoop/yarn/service/webapp/ApiServer.java   |  34 +-
 .../yarn/service/webapp/package-info.java   |  28 --
 .../service/client/TestApiServiceClient.java| 259 ---
 .../src/test/resources/example-app.json |  15 -
 .../src/test/resources/log4j.properties |  19 -
 .../service/api/records/ReadinessCheck.java |   2 -
 .../yarn/service/api/records/Resource.java  |   1 +
 .../yarn/service/api/records/ServiceState.java  |   2 +-
 .../hadoop/yarn/service/utils/JsonSerDeser.java |   3 -
 .../client/TestBuildExternalComponents.java |   2 +-
 .../yarn/service/client/TestServiceCLI.java |  14 +-
 .../hadoop/yarn/client/api/AppAdminClient.java  |   6 -
 14 files changed, 10 insertions(+), 814 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/836e3c45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index bae62c6..ddea2a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -65,15 +65,6 @@
   
 
   
-  
-org.apache.rat
-apache-rat-plugin
-
-  
-**/*.json
-  
-
-  
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/836e3c45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
deleted file mode 100644
index f4133a5..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.service.client;
-
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.jsonSerDeser;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.List;
-import java.util.Map;
-
-import javax.ws.rs.core.MediaType;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import 

hadoop git commit: YARN-7565. Yarn service pre-maturely releases the container after AM restart. Contributed by Chandni Singh

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 06f0eb2dc -> 3ebe6a781


YARN-7565. Yarn service pre-maturely releases the container after AM restart. 
Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ebe6a78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ebe6a78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ebe6a78

Branch: refs/heads/trunk
Commit: 3ebe6a7819292ce6bd557e36137531b59890c845
Parents: 06f0eb2
Author: Jian He 
Authored: Tue Dec 12 13:35:56 2017 -0800
Committer: Jian He 
Committed: Tue Dec 12 13:35:56 2017 -0800

--
 .../hadoop/yarn/service/ServiceScheduler.java   |  72 +-
 .../yarn/service/component/Component.java   |   8 ++
 .../component/instance/ComponentInstance.java   |  10 +-
 .../yarn/service/conf/YarnServiceConf.java  |  10 ++
 .../hadoop/yarn/service/MockServiceAM.java  | 144 ---
 .../hadoop/yarn/service/TestServiceAM.java  |  98 -
 .../yarn/client/api/async/AMRMClientAsync.java  |  10 ++
 .../api/async/impl/AMRMClientAsyncImpl.java |   5 +
 .../types/yarn/YarnRegistryAttributes.java  |   1 +
 .../markdown/yarn-service/Configurations.md |   1 +
 10 files changed, 330 insertions(+), 29 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ebe6a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index bea31cf..2697050 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -65,6 +65,7 @@ import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventT
 import org.apache.hadoop.yarn.service.component.Component;
 import org.apache.hadoop.yarn.service.component.ComponentEvent;
 import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
 import org.apache.hadoop.yarn.service.provider.ProviderUtils;
@@ -137,6 +138,9 @@ public class ServiceScheduler extends CompositeService {
   private YarnRegistryViewForProviders yarnRegistryOperations;
   private ServiceContext context;
   private ContainerLaunchService containerLaunchService;
+  private final Map unRecoveredInstances =
+  new ConcurrentHashMap<>();
+  private long containerRecoveryTimeout;
 
   public ServiceScheduler(ServiceContext context) {
 super(context.service.getName());
@@ -212,6 +216,9 @@ public class ServiceScheduler extends CompositeService {
 createConfigFileCache(context.fs.getFileSystem());
 
 createAllComponents();
+containerRecoveryTimeout = getConfig().getInt(
+YarnServiceConf.CONTAINER_RECOVERY_TIMEOUT_MS,
+YarnServiceConf.DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS);
   }
 
   protected YarnRegistryViewForProviders createYarnRegistryOperations(
@@ -320,7 +327,7 @@ public class ServiceScheduler extends CompositeService {
 }
 for (Container container : containersFromPrevAttempt) {
   LOG.info("Handling {} from previous attempt", container.getId());
-  ServiceRecord record = existingRecords.get(RegistryPathUtils
+  ServiceRecord record = existingRecords.remove(RegistryPathUtils
   .encodeYarnID(container.getId().toString()));
   if (record != null) {
 Component comp = 
componentsById.get(container.getAllocationRequestId());
@@ -337,6 +344,40 @@ public class ServiceScheduler extends CompositeService {
 amRMClient.releaseAssignedContainer(container.getId());
   }
 }
+
+existingRecords.forEach((encodedContainerId, record) -> {
+  String componentName = record.get(YarnRegistryAttributes.YARN_COMPONENT);
+  if (componentName != null) {
+Component component = componentsByName.get(componentName);
+ComponentInstance compInstance = 

hadoop git commit: YARN-6168. Restarted RM may not inform AM about all existing containers. Contributed by Chandni Singh

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 3cd75845d -> fedabcad4


YARN-6168. Restarted RM may not inform AM about all existing containers. 
Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fedabcad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fedabcad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fedabcad

Branch: refs/heads/trunk
Commit: fedabcad42067ac7dd24de40fab6be2d3485a540
Parents: 3cd7584
Author: Jian He 
Authored: Mon Nov 27 09:55:08 2017 -0800
Committer: Jian He 
Committed: Mon Nov 27 10:19:58 2017 -0800

--
 .../api/protocolrecords/AllocateResponse.java   |  54 +++
 .../src/main/proto/yarn_service_protos.proto|   1 +
 .../impl/pb/AllocateResponsePBImpl.java |  37 +
 .../resourcemanager/DefaultAMSProcessor.java|   3 +
 .../scheduler/AbstractYarnScheduler.java|   4 +-
 .../resourcemanager/scheduler/Allocation.java   |  13 +-
 .../scheduler/SchedulerApplicationAttempt.java  |  48 ++
 .../scheduler/common/fica/FiCaSchedulerApp.java |   5 +-
 .../scheduler/fair/FairScheduler.java   |   3 +-
 .../applicationsmanager/TestAMRestart.java  | 149 +++
 10 files changed, 310 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fedabcad/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index 9b254ae..98346ce 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -372,6 +372,44 @@ public abstract class AllocateResponse {
   public void setUpdateErrors(List updateErrors) {
   }
 
+  /**
+   * Get the list of running containers as viewed by
+   * ResourceManager from previous application attempts which
+   * have not been reported to the Application Master yet.
+   * 
+   * These containers were recovered by the RM after the application master
+   * had already registered. This may happen after RM restart when some NMs get
+   * delayed in connecting to the RM and reporting the active containers.
+   * Since they were not reported in the registration
+   * response, they are reported in the response to the AM heartbeat.
+   *
+   * @return the list of running containers as viewed by
+   * ResourceManager from previous application attempts.
+   */
+  @Public
+  @Unstable
+  public abstract List getContainersFromPreviousAttempts();
+
+  /**
+   * Set the list of running containers as viewed by
+   * ResourceManager from previous application attempts which have
+   * not been reported to the Application Master yet.
+   * 
+   * These containers were recovered by the RM after the application master
+   * had already registered. This may happen after RM restart when some NMs get
+   * delayed in connecting to the RM and reporting the active containers.
+   * Since they were not reported in the registration
+   * response, they are reported in the response to the AM heartbeat.
+   *
+   * @param containersFromPreviousAttempt
+   *  the list of running containers as viewed by
+   *  ResourceManager from previous application attempts.
+   */
+  @Private
+  @Unstable
+  public abstract void setContainersFromPreviousAttempts(
+  List containersFromPreviousAttempt);
+
   @Private
   @Unstable
   public static AllocateResponseBuilder newBuilder() {
@@ -590,6 +628,22 @@ public abstract class AllocateResponse {
 }
 
 /**
+ * Set the containersFromPreviousAttempt of the response.
+ * @see AllocateResponse#setContainersFromPreviousAttempts(List)
+ * @param containersFromPreviousAttempt
+ * containersFromPreviousAttempt of the response
+ * @return {@link AllocateResponseBuilder}
+ */
+@Private
+@Unstable
+public AllocateResponseBuilder containersFromPreviousAttempt(
+List containersFromPreviousAttempt) {
+  allocateResponse.setContainersFromPreviousAttempts(
+  containersFromPreviousAttempt);
+  return this;
+}
+
+/**
  * Return generated {@link AllocateResponse} object.
  * @return {@link AllocateResponse}
  */

hadoop git commit: YARN-7503. Configurable heap size / JVM opts in service AM. Contributed by Jonathan Hung

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 28d0fcbef -> 6bf2c3019


YARN-7503. Configurable heap size / JVM opts in service AM. Contributed by 
Jonathan Hung


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bf2c301
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bf2c301
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bf2c301

Branch: refs/heads/trunk
Commit: 6bf2c301924a3acae5a7510b8473f6292a5a471b
Parents: 28d0fcb
Author: Jian He 
Authored: Thu Nov 16 10:53:55 2017 -0800
Committer: Jian He 
Committed: Thu Nov 16 10:53:55 2017 -0800

--
 .../apache/hadoop/yarn/service/client/ServiceClient.java | 10 +-
 .../apache/hadoop/yarn/service/conf/YarnServiceConf.java |  5 +
 .../service/containerlaunch/JavaCommandLineBuilder.java  | 11 +--
 3 files changed, 15 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index af43f8a..d1b6026 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -559,7 +559,7 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 Map env = addAMEnv();
 
 // create AM CLI
-String cmdStr = buildCommandLine(serviceName, conf, appRootDir, 
hasAMLog4j);
+String cmdStr = buildCommandLine(app, conf, appRootDir, hasAMLog4j);
 submissionContext.setResource(Resource.newInstance(YarnServiceConf
 .getLong(YarnServiceConf.AM_RESOURCE_MEM,
 YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, 
app.getConfiguration(),
@@ -624,12 +624,12 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 LOG.debug(builder.toString());
   }
 
-  private String buildCommandLine(String serviceName, Configuration conf,
+  private String buildCommandLine(Service app, Configuration conf,
   Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
 JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
 CLI.forceIPv4().headless();
-//TODO CLI.setJVMHeap
-//TODO CLI.addJVMOPTS
+CLI.setJVMOpts(YarnServiceConf.get(YarnServiceConf.JVM_OPTS, null,
+app.getConfiguration(), conf));
 if (hasSliderAMLog4j) {
   CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
   CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
@@ -637,7 +637,7 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 CLI.add(ServiceMaster.class.getCanonicalName());
 //TODO debugAM CLI.add(Arguments.ARG_DEBUG)
 CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir,
-serviceName + ".json"));
+app.getName() + ".json"));
 // pass the registry binding
 CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
 RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index a7bd58d..684d980 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ 

hadoop git commit: YARN-7488. Make ServiceClient.getAppId method public to return ApplicationId for a service name. Contributed by Gour Saha

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 8b1257416 -> 4f40cd314


YARN-7488. Make ServiceClient.getAppId method public to return ApplicationId 
for a service name. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f40cd31
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f40cd31
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f40cd31

Branch: refs/heads/trunk
Commit: 4f40cd314ab14f735a465fb9dff2dc1bf118e703
Parents: 8b12574
Author: Jian He 
Authored: Mon Nov 13 18:55:12 2017 -0800
Committer: Jian He 
Committed: Mon Nov 13 18:57:56 2017 -0800

--
 .../java/org/apache/hadoop/yarn/service/client/ServiceClient.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f40cd31/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 11cd30d..af43f8a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -943,7 +943,7 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 UserGroupInformation.getCurrentUser(), rpc, address);
   }
 
-  private synchronized ApplicationId getAppId(String serviceName)
+  public synchronized ApplicationId getAppId(String serviceName)
   throws IOException, YarnException {
 if (cachedAppIds.containsKey(serviceName)) {
   return cachedAppIds.get(serviceName);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-7466. ResourceRequest has a different default for allocationRequestId than Container. Contributed by Chandni Singh

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk e14f03dfb -> 5323b0048


YARN-7466. ResourceRequest has a different default for allocationRequestId than 
Container. Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5323b004
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5323b004
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5323b004

Branch: refs/heads/trunk
Commit: 5323b0048b743771276ba860b10c27b23a70bf9e
Parents: e14f03d
Author: Jian He 
Authored: Mon Nov 13 15:37:39 2017 -0800
Committer: Jian He 
Committed: Mon Nov 13 15:37:39 2017 -0800

--
 .../hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5323b004/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index e69c07b..7769c48 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -388,7 +388,7 @@ message ResourceRequestProto {
   optional bool relax_locality = 5 [default = true];
   optional string node_label_expression = 6;
   optional ExecutionTypeRequestProto execution_type_request = 7;
-  optional int64 allocation_request_id = 8 [default = 0];
+  optional int64 allocation_request_id = 8 [default = -1];
   optional ProfileCapabilityProto profile = 9;
 }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[43/60] [abbrv] hadoop git commit: Revert "YARN-6626. Embed REST API service into RM. Contributed by Eric Yang"

[jianhe]

Revert "YARN-6626. Embed REST API service into RM. Contributed by Eric Yang"

This reverts commit 63d1084e9781e0fee876916190b69f6242dd00e4.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/673c0db4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/673c0db4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/673c0db4

Branch: refs/heads/trunk
Commit: 673c0db43cfa4a0e9e7a21c5ea31660ecf61b8c3
Parents: 9e677fa
Author: Jian He 
Authored: Wed Oct 4 23:39:27 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:17 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  2 --
 .../hadoop/yarn/service/webapp/ApiServer.java   |  7 
 .../hadoop-yarn-services-core/pom.xml   |  6 
 .../yarn/service/api/records/Artifact.java  |  8 -
 .../yarn/service/api/records/ConfigFile.java|  8 -
 .../service/api/records/ReadinessCheck.java |  8 -
 .../src/main/resources/yarn-default.xml |  8 -
 .../server/resourcemanager/webapp/RMWebApp.java | 19 ---
 .../site/markdown/yarn-service/QuickStart.md| 34 +++-
 9 files changed, 11 insertions(+), 89 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4799137..640e86e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,8 +336,6 @@ public class YarnConfiguration extends Configuration {
 
   public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
   + "webapp.ui2.war-file-path";
-  public static final String YARN_API_SERVICES_ENABLE = "yarn."
-  + "webapp.api-service.enable";
 
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
 RM_PREFIX + "resource-tracker.address";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 89b020d..e8286ef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,7 +17,6 @@
 
 package org.apache.hadoop.yarn.service.webapp;
 
-import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
@@ -58,12 +57,6 @@ import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 @Singleton
 @Path(CONTEXT_ROOT)
 public class ApiServer {
-
-  @Inject
-  public ApiServer(Configuration conf) {
-super();
-  }
-
   private static final Logger LOG =
   LoggerFactory.getLogger(ApiServer.class);
   private static Configuration YARN_CONFIG = new YarnConfiguration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/673c0db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 205a64d..fb07edc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,6 +245,12 @@
 
 
 
+  org.apache.hadoop
+  hadoop-yarn-server-resourcemanager
+  test
+
+
+
   

[59/60] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He

[jianhe]

YARN-6394. Support specifying YARN related params in the service REST API. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6c4bd74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6c4bd74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6c4bd74

Branch: refs/heads/trunk
Commit: a6c4bd74b636d0715ffb4c1aca9ef4e49201aa02
Parents: a288b0a
Author: Billie Rinaldi 
Authored: Fri Nov 3 11:25:57 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:20 2017 -0800

--
 .../definition/YARN-Services-Examples.md|   8 +-
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |   6 +-
 .../examples/httpd-no-dns/httpd-no-dns.json |   6 +-
 .../examples/httpd/httpd.json   |   4 +-
 .../hadoop/yarn/service/ServiceMaster.java  |   6 +
 .../hadoop/yarn/service/ServiceMetrics.java |   4 -
 .../hadoop/yarn/service/ServiceScheduler.java   |  15 --
 .../yarn/service/api/ServiceApiConstants.java   |   4 +-
 .../yarn/service/api/records/ConfigFile.java|  32 ++--
 .../yarn/service/api/records/Configuration.java |   4 +-
 .../service/api/records/ReadinessCheck.java |  24 +--
 .../yarn/service/client/ServiceClient.java  |  61 ---
 .../yarn/service/component/Component.java   |  29 ++--
 .../component/instance/ComponentInstance.java   |  37 +++--
 .../yarn/service/conf/YarnServiceConf.java  |  14 +-
 .../yarn/service/conf/YarnServiceConstants.java |   2 +
 .../service/monitor/probe/MonitorUtils.java |   4 +-
 .../provider/AbstractClientProvider.java|  17 +-
 .../yarn/service/provider/ProviderUtils.java|  12 +-
 .../timelineservice/ServiceTimelineEvent.java   |   4 +-
 .../ServiceTimelinePublisher.java   |  48 +++---
 .../yarn/service/conf/TestAppJsonResolve.java   |   8 +-
 .../TestServiceTimelinePublisher.java   |   2 +-
 .../service/conf/examples/app-override.json |   6 +-
 .../markdown/yarn-service/Configurations.md | 165 +++
 .../src/site/markdown/yarn-service/Examples.md  |   4 +-
 .../src/site/markdown/yarn-service/Overview.md  |   3 +-
 .../markdown/yarn-service/YarnServiceAPI.md |  14 +-
 .../app/controllers/yarn-app/components.js  |   2 +-
 .../app/templates/components/metrics-table.hbs  |   2 +-
 .../templates/yarn-component-instance/info.hbs  |   4 +-
 .../templates/yarn-component-instances/info.hbs |   2 +-
 32 files changed, 384 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
index 3cd3d48..00b21dd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -176,14 +176,14 @@ POST URL - 
http://localhost:9191:/ws/v1/services/hbase-app-1
   {
 "type": "XML",
 "dest_file": "/etc/hadoop/conf/core-site.xml",
-"props": {
+"properties": {
   "fs.defaultFS": "${CLUSTER_FS_URI}"
 }
   },
   {
 "type": "XML",
 "dest_file": "/etc/hbase/conf/hbase-site.xml",
-"props": {
+"properties": {
   "hbase.cluster.distributed": "true",
   "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
   "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
@@ -216,14 +216,14 @@ POST URL - 
http://localhost:9191:/ws/v1/services/hbase-app-1
   {
 "type": "XML",
 "dest_file": "/etc/hadoop/conf/core-site.xml",
-"props": {
+"properties": {
   "fs.defaultFS": "${CLUSTER_FS_URI}"
 }
   },
   {
 "type": "XML",
 "dest_file": "/etc/hbase/conf/hbase-site.xml",
-"props": {
+"properties": {
   "hbase.cluster.distributed": "true",
   "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
   "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",

[49/60] [abbrv] hadoop git commit: YARN-7393. RegistryDNS doesn't work in tcp channel. Contributed by Eric Yang

[jianhe]

YARN-7393. RegistryDNS doesn't work in tcp channel. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ec9e72c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ec9e72c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ec9e72c

Branch: refs/heads/trunk
Commit: 8ec9e72cc37957d8cf9af5a7b78086441ae64b00
Parents: d2775fb
Author: Billie Rinaldi 
Authored: Fri Oct 27 11:44:55 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:19 2017 -0800

--
 .../hadoop/registry/server/dns/RegistryDNS.java | 20 +++-
 1 file changed, 15 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ec9e72c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 9097c46..d7ebece 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,8 +189,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
 
 LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
 addNIOUDP(addr, port);
-//TODO Fix dns lookup over TCP
-//addNIOTCP(addr, port);
+addNIOTCP(addr, port);
   }
 
   /**
@@ -768,7 +767,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
   buf.get(in, 0, messageLength);
 
   Message query;
-  byte[] response = null;
+  byte[] response;
   try {
 query = new Message(in);
 LOG.info("received TCP query {}", query.getQuestion());
@@ -781,10 +780,19 @@ public class RegistryDNS extends AbstractService 
implements DNSOperations,
   }
 
   ByteBuffer out = ByteBuffer.allocate(response.length + 2);
-  out.putShort(0, (short) (response.length & 0x));
+  out.clear();
+  byte[] data = new byte[2];
+
+  data[1] = (byte)(response.length & 0xFF);
+  data[0] = (byte)((response.length >> 8) & 0xFF);
+  out.put(data);
   out.put(response);
+  out.flip();
+
+  while(out.hasRemaining()) {
+ch.write(out);
+  }
 
-  ch.write(out);
 } catch (IOException e) {
   throw NetUtils.wrapException(ch.socket().getInetAddress().getHostName(),
   ch.socket().getPort(),
@@ -836,6 +844,8 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
 }
   });
 
+} else {
+  Thread.sleep(500);
 }
   }
 } catch (IOException e) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[52/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
deleted file mode 100644
index 65f0472..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionResolveArgs.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-
-import java.io.File;
-
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_RESOLVE;
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-
-/**
- * Resolve registry entries
- * 
- * --path {path}
- * --out {destfile}
- * --verbose
- * --list
- */
-@Parameters(commandNames = {ACTION_RESOLVE},
-commandDescription = DESCRIBE_ACTION_REGISTRY)
-public class ActionResolveArgs extends AbstractActionArgs {
-
-  public static final String USAGE =
-  "Usage: " + SliderActions.ACTION_RESOLVE
-  + " "
-  + ARG_PATH + "  "
-  + "[" + ARG_LIST + "] "
-  + "[" + ARG_OUTPUT + "  ] "
-  + "[" + ARG_DESTDIR + "  ] "
-  ;
-  public ActionResolveArgs() {
-  }
-
-  @Override
-  public String getActionName() {
-return ACTION_RESOLVE;
-  }
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  @Override
-  public int getMinParams() {
-return 0;
-  }
-  
-  @Parameter(names = {ARG_LIST}, 
-  description = "list services")
-  public boolean list;
-
-  @Parameter(names = {ARG_PATH},
-  description = "resolve a path")
-  public String path;
-
-  @Parameter(names = {ARG_DESTDIR},
-  description = "destination directory for operations")
-  public File destdir;
-
-  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
-  description = "dest file")
-  public File out;
-
-  @Override
-  public String toString() {
-final StringBuilder sb =
-new StringBuilder(ACTION_RESOLVE).append(" ");
-sb.append(ARG_PATH).append(" ").append(path).append(" ");
-if (list) {
-  sb.append(ARG_LIST).append(" ");
-}
-if (destdir != null) {
-  sb.append(ARG_DESTDIR).append(" ").append(destdir).append(" ");
-}
-if (out != null) {
-  sb.append(ARG_OUTPUT).append(" ").append(out).append(" ");
-}
-return sb.toString();
-  }
-
-  @Override
-  public void validate() throws BadCommandArgumentsException, UsageException {
-super.validate();
-if (StringUtils.isEmpty(path)) {
-  throw new BadCommandArgumentsException("Missing mandatory argument "
- + ARG_PATH);
-}
-if (list && out != null) {
-  throw new BadCommandArgumentsException("Argument "
- + ARG_OUTPUT +
- " not supported for " + ARG_LIST);
-}
-if (out != null && destdir != null) {
-  throw new BadCommandArgumentsException(
-  ARG_OUTPUT + " and " + ARG_DESTDIR + " cannot be used together"
-  );
-}
-  }
-
-  public String getPath() {
-return path;
-  }
-
-  public void setPath(String path) {
-this.path = path;
-  }
-
-  public boolean isList() {
-return list;
- 

[54/60] [abbrv] hadoop git commit: YARN-7326. Add recursion support and configure RegistryDNS to lookup upstream. Contributed by Eric Yang

[jianhe]

YARN-7326. Add recursion support and configure RegistryDNS to lookup upstream. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/501be9b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/501be9b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/501be9b4

Branch: refs/heads/trunk
Commit: 501be9b4be7d549b23cc501b26fdfdce0a7aa911
Parents: 68acd88
Author: Billie Rinaldi 
Authored: Wed Oct 25 07:37:03 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:19 2017 -0800

--
 .../hadoop/registry/server/dns/RegistryDNS.java | 148 ++-
 .../registry/server/dns/TestRegistryDNS.java|   8 +-
 2 files changed, 148 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/501be9b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index d7a415d..37e8429 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -37,8 +37,10 @@ import org.xbill.DNS.DNSKEYRecord;
 import org.xbill.DNS.DNSSEC;
 import org.xbill.DNS.DSRecord;
 import org.xbill.DNS.ExtendedFlags;
+import org.xbill.DNS.ExtendedResolver;
 import org.xbill.DNS.Flags;
 import org.xbill.DNS.Header;
+import org.xbill.DNS.Lookup;
 import org.xbill.DNS.Message;
 import org.xbill.DNS.NSRecord;
 import org.xbill.DNS.Name;
@@ -49,9 +51,12 @@ import org.xbill.DNS.RRSIGRecord;
 import org.xbill.DNS.RRset;
 import org.xbill.DNS.Rcode;
 import org.xbill.DNS.Record;
+import org.xbill.DNS.Resolver;
+import org.xbill.DNS.ResolverConfig;
 import org.xbill.DNS.SOARecord;
 import org.xbill.DNS.Section;
 import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.SimpleResolver;
 import org.xbill.DNS.TSIG;
 import org.xbill.DNS.TSIGRecord;
 import org.xbill.DNS.TextParseException;
@@ -66,8 +71,11 @@ import java.io.IOException;
 import java.math.BigInteger;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
 import java.net.Socket;
 import java.net.SocketAddress;
+import java.net.SocketException;
+import java.net.UnknownHostException;
 import java.nio.ByteBuffer;
 import java.nio.channels.DatagramChannel;
 import java.nio.channels.ServerSocketChannel;
@@ -78,10 +86,13 @@ import java.security.PrivateKey;
 import java.security.spec.InvalidKeySpecException;
 import java.security.spec.RSAPrivateKeySpec;
 import java.text.SimpleDateFormat;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Collection;
 import java.util.Date;
+import java.util.Enumeration;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.Callable;
@@ -134,6 +145,16 @@ public class RegistryDNS extends AbstractService 
implements DNSOperations,
   private boolean channelsInitialized = false;
 
   /**
+   * Lock to update resolver only once per request.
+   */
+  private final Object resolverUpdateLock = new Object();
+
+  /**
+   * Whether resolver update has been requested.
+   */
+  private boolean resolverUpdateRequested = true;
+
+  /**
* Construct the service.
*
* @param name service name
@@ -172,6 +193,79 @@ public class RegistryDNS extends AbstractService 
implements DNSOperations,
   }
 
   /**
+   * Initialize registryDNS to use /etc/resolv.conf values
+   * as default resolvers.
+   */
+  private void updateDNSServer(Configuration conf) {
+synchronized (resolverUpdateLock) {
+  if (!resolverUpdateRequested) {
+return;
+  }
+  int port = conf.getInt(KEY_DNS_PORT, DEFAULT_DNS_PORT);
+  resolverUpdateRequested = false;
+  List list = new ArrayList();
+  try {
+// If resolv.conf contains the server's own IP address,
+// and RegistryDNS handles the lookup.  Local IP address
+// must be filter out from default resolvers to prevent
+// self recursive loop.
+if (port != 53) {
+  // When registryDNS is not running on default port,
+  // registryDNS can utilize local DNS server as upstream lookup.
+  throw new SocketException("Bypass filtering local DNS server.");
+}
+Enumeration net =
+

[33/60] [abbrv] hadoop git commit: YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha

[jianhe]

YARN-7161. Fix special chars in DNS documentation. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9ca13e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9ca13e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9ca13e3

Branch: refs/heads/trunk
Commit: b9ca13e34bc8a14787e94e9b70ae0aa249f00954
Parents: eed9e3e
Author: Jian He 
Authored: Thu Sep 7 18:17:02 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:15 2017 -0800

--
 .../native-services/NativeServicesDiscovery.md  | 87 ++--
 1 file changed, 45 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9ca13e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
index fa54a09..a927118 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -13,39 +13,42 @@
 -->
 
 # YARN DNS Server
+
+
+
 ## Introduction
 
 The YARN DNS Server provides a standard DNS interface to the information 
posted into the YARN Registry by deployed applications. The DNS service serves 
the following functions:
 
-1. **Exposing existing service­ discovery information via DNS**​­ - 
Information provided in
+1. **Exposing existing service-discovery information via DNS** - Information 
provided in
 the current YARN service registry’s records will be converted into DNS 
entries, thus
 allowing users to discover information about YARN applications using standard 
DNS
 client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
 number for services).
-2. **Enabling Container to IP mappings​­** - Enables discovery of the IPs 
of containers via
+2. **Enabling Container to IP mappings** - Enables discovery of the IPs of 
containers via
 standard DNS lookups. Given the availability of the records via DNS, container
-name­based communication will be facilitated (e.g. ‘curl
+name-based communication will be facilitated (e.g. ‘curl
 http://myContainer.myDomain.com/endpoint’).
 
 ## Service Properties
 
 The existing YARN Service Registry is leveraged as the source of information 
for the DNS Service.
 
-The following core functions are supported by the DNS­ Server:
+The following core functions are supported by the DNS-Server:
 
-###Functional properties
+### Functional properties
 
-1. Supports creation of DNS records for end­points of the deployed YARN 
applications
+1. Supports creation of DNS records for end-points of the deployed YARN 
applications
 2. Record names remain unchanged during restart of containers and/or 
applications
 3. Supports reverse lookups (name based on IP).
 4. Supports security using the standards defined by The Domain Name System 
Security
 Extensions (DNSSEC)
 5. Highly available
-6. Scalable ­- The service provides the responsiveness (e.g. low­ latency) 
required to
+6. Scalable - The service provides the responsiveness (e.g. low-latency) 
required to
 respond to DNS queries (timeouts yield attempts to invoke other configured name
 servers).
 
-###Deployment properties
+### Deployment properties
 
 1. Supports integration with existing DNS assets (e.g. a corporate DNS server) 
by acting as
 a DNS server for a Hadoop cluster zone/domain. The server is not intended to 
act as a
@@ -55,59 +58,59 @@ DNS standards. The default port for DNS protocols is in a 
restricted, administra
 range (53), so the port is configurable for deployments in which the service 
may
 not be managed via an administrative account.
 
-##DNS Record Name Structure
+## DNS Record Name Structure
 
 The DNS names of generated records are composed from the following elements 
(labels). Note that these elements must be compatible with DNS conventions (see 
“Preferred Name Syntax” in RFC 1035):
 
-* **domain** -​­ the name of the cluster DNS domain. This name is provided 
as a
+* **domain** - the name of the cluster DNS domain. This name is provided as a
 configuration property. In addition, it is this name that is configured at a 
parent DNS
 server as the zone name for the defined yDNS zone (the zone for which the 
parent DNS
 server will forward requests to yDNS). E.g. yarncluster.com
-* **user­name**​ -­ the name of the application deployer. This name is the 

[36/60] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He

[jianhe]

YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/541b64c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/541b64c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/541b64c7

Branch: refs/heads/trunk
Commit: 541b64c7985db425c5ce8bd5f71a2d5980809708
Parents: b9ca13e
Author: Gour Saha 
Authored: Thu Sep 7 23:25:31 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:15 2017 -0800

--
 hadoop-project/pom.xml  |   4 +-
 hadoop-project/src/site/site.xml|  10 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   7 +-
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  11 +-
 .../hadoop/yarn/service/ServiceMaster.java  |  10 +-
 .../hadoop/yarn/service/client/ServiceCLI.java  |   5 +-
 .../yarn/service/client/ServiceClient.java  |  20 +-
 .../client/params/AbstractActionArgs.java   |  33 +-
 .../AbstractClusterBuildingActionArgs.java  |  22 +-
 .../service/client/params/ActionBuildArgs.java  |   5 +
 .../service/client/params/ActionCreateArgs.java |   5 +
 .../client/params/ActionDependencyArgs.java |   4 +-
 .../service/client/params/ActionExamples.java   |  26 +
 .../service/client/params/ActionUpdateArgs.java |   7 +-
 .../yarn/service/client/params/Arguments.java   |   9 +-
 .../yarn/service/client/params/ClientArgs.java  |  10 -
 .../yarn/service/client/params/CommonArgs.java  |  28 +-
 .../service/client/params/ServiceAMArgs.java|  50 ++
 .../client/params/ServiceAMCreateAction.java|  37 ++
 .../service/client/params/SliderAMArgs.java |  57 --
 .../client/params/SliderAMCreateAction.java |  73 ---
 .../service/client/params/SliderActions.java|   9 +-
 .../ServiceTimelinePublisher.java   |   5 +-
 .../client/TestBuildExternalComponents.java |   6 +-
 .../yarn/service/client/TestServiceCLI.java |  30 +-
 .../hadoop/yarn/service/conf/examples/app.json  |   2 +-
 .../registry/client/api/RegistryConstants.java  |   2 +-
 .../src/site/markdown/YarnCommands.md   |  56 +-
 .../native-services/NativeServicesAPI.md| 606 ---
 .../native-services/NativeServicesDiscovery.md  | 144 -
 .../native-services/NativeServicesIntro.md  | 107 
 .../src/site/markdown/yarn-service/Concepts.md  |  77 +++
 .../src/site/markdown/yarn-service/Overview.md  |  58 ++
 .../site/markdown/yarn-service/QuickStart.md| 218 +++
 .../markdown/yarn-service/ServiceDiscovery.md   | 150 +
 .../markdown/yarn-service/YarnServiceAPI.md | 592 ++
 36 files changed, 1363 insertions(+), 1132 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3cb9568..d18c9a8 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -143,10 +143,10 @@
 
 ${project.version}
-
+
 1.5.4
 1.16
-1.30
+1.48
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index ad893a2..57cff9a 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -155,10 +155,12 @@
   
 
 
-
-  
-  
-  
+
+  
+  
+  
+  
+  
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 1db585e..05d892b 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -31,7 +31,7 @@ function hadoop_usage
   hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
   hadoop_add_option "--workers" "turn on worker mode"
 
-  hadoop_add_subcommand "apiserver" "run yarn-native-service rest server"
+  hadoop_add_subcommand "apiserver" daemon "run the api-server for 
deploying/managing services on YARN"
   hadoop_add_subcommand "application" client "prints application(s) 
report/kill application"
   hadoop_add_subcommand "applicationattempt" client "prints 
applicationattempt(s) report"
   hadoop_add_subcommand "classpath" client "prints the class path needed to 
get the hadoop jar and the required libraries"
@@ -50,10 +50,9 @@ function hadoop_usage
   hadoop_add_subcommand "router" daemon "run the Router daemon"
   

[46/60] [abbrv] hadoop git commit: YARN-7202. Add UT for api-server. Contributed by Eric Yang

[jianhe]

YARN-7202. Add UT for api-server. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57144a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57144a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57144a1

Branch: refs/heads/trunk
Commit: b57144a1726e0b02abbdc3e8f37769d3f7c039c8
Parents: b8a7ef1
Author: Jian He 
Authored: Thu Oct 12 10:57:35 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 .../hadoop-yarn-services-api/pom.xml|  34 +-
 .../hadoop/yarn/service/webapp/ApiServer.java   |  21 +-
 .../yarn/service/webapp/ApiServerWebApp.java|   2 +-
 .../hadoop/yarn/service/ServiceClientTest.java  | 107 ++
 .../hadoop/yarn/service/TestApiServer.java  | 366 +++
 .../hadoop/yarn/service/ServiceTestUtils.java   | 171 +
 .../yarn/service/TestYarnNativeServices.java| 162 +---
 7 files changed, 690 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 74d9681..ddea2a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -41,7 +41,6 @@
 
 
 
-
   
 org.apache.maven.plugins
 maven-jar-plugin
@@ -66,17 +65,6 @@
   
 
   
-
-  
-org.apache.maven.plugins
-maven-surefire-plugin
-
-  
-${java.home}
-  
-
-  
-
 
   
 
@@ -84,6 +72,7 @@
   
 
   
+
 
   org.apache.hadoop
   hadoop-yarn-services-core
@@ -116,5 +105,26 @@
   javax.ws.rs
   jsr311-api
 
+
+  org.mockito
+  mockito-all
+  test
+
+
+
+
+
+
+
+  org.apache.hadoop
+  hadoop-common
+  test-jar
+
+
+  junit
+  junit
+  test
+
+
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 4fe12e3..5773069 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -165,12 +165,12 @@ public class ApiServer {
   } else {
 LOG.info("Successfully stopped service {}", appName);
   }
-  return Response.status(Status.NO_CONTENT).build();
+  return Response.status(Status.OK).build();
 } catch (ApplicationNotFoundException e) {
   ServiceStatus serviceStatus = new ServiceStatus();
   serviceStatus.setDiagnostics(
-  "Service " + appName + " not found " + e.getMessage());
-  return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+  "Service " + appName + " is not found in YARN: " + e.getMessage());
+  return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
   .build();
 } catch (Exception e) {
   ServiceStatus serviceStatus = new ServiceStatus();
@@ -245,7 +245,8 @@ public class ApiServer {
 // flex a single component app
 if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
 .hasComponent(updateServiceData)) {
-  Component defaultComp = 
ServiceApiUtil.createDefaultComponent(updateServiceData);
+  Component defaultComp = ServiceApiUtil
+  .createDefaultComponent(updateServiceData);
   return updateComponent(updateServiceData.getName(), 
defaultComp.getName(),
   defaultComp);
 }
@@ -291,4 +292,16 @@ public class ApiServer {
   .entity(status).build();
 }
   }
+
+  /**
+   * Used by negative test case.
+   *
+   * @param mockServerClient - A mocked version of ServiceClient
+   */
+  

[40/60] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang

[jianhe]

YARN-6626. Embed REST API service into RM. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7230215
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7230215
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7230215

Branch: refs/heads/trunk
Commit: c72302157907b44b71bc15008aabd9cf3d806f12
Parents: 673c0db
Author: Jian He 
Authored: Thu Sep 28 16:29:22 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:17 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  2 ++
 .../hadoop/yarn/service/webapp/ApiServer.java   | 11 +++
 .../hadoop-yarn-services-core/pom.xml   |  6 
 .../yarn/service/api/records/Artifact.java  |  8 +
 .../yarn/service/api/records/ConfigFile.java|  8 +
 .../service/api/records/ReadinessCheck.java |  8 +
 .../src/main/resources/yarn-default.xml |  8 +
 .../server/resourcemanager/webapp/RMWebApp.java | 19 +++
 .../site/markdown/yarn-service/QuickStart.md| 34 +---
 9 files changed, 93 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 640e86e..4799137 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration {
 
   public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
   + "webapp.ui2.war-file-path";
+  public static final String YARN_API_SERVICES_ENABLE = "yarn."
+  + "webapp.api-service.enable";
 
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
 RM_PREFIX + "resource-tracker.address";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e8286ef..4fe12e3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.yarn.service.webapp;
 
+import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,16 @@ import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 @Singleton
 @Path(CONTEXT_ROOT)
 public class ApiServer {
+
+  public ApiServer() {
+super();
+  }
+  
+  @Inject
+  public ApiServer(Configuration conf) {
+super();
+  }
+
   private static final Logger LOG =
   LoggerFactory.getLogger(ApiServer.class);
   private static Configuration YARN_CONFIG = new YarnConfiguration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7230215/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index fb07edc..205a64d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,12 +245,6 @@
 
 
 
-  org.apache.hadoop
-  hadoop-yarn-server-resourcemanager
-  test
-
-
-
   org.apache.curator

[42/60] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang

[jianhe]

YARN-6626. Embed REST API service into RM. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e677fa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e677fa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e677fa0

Branch: refs/heads/trunk
Commit: 9e677fa05c6a822cf034bffd25bbee0bad84084b
Parents: 8851209
Author: Jian He 
Authored: Thu Sep 28 16:29:22 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:17 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  2 ++
 .../hadoop/yarn/service/webapp/ApiServer.java   |  7 
 .../hadoop-yarn-services-core/pom.xml   |  6 
 .../yarn/service/api/records/Artifact.java  |  8 +
 .../yarn/service/api/records/ConfigFile.java|  8 +
 .../service/api/records/ReadinessCheck.java |  8 +
 .../src/main/resources/yarn-default.xml |  8 +
 .../server/resourcemanager/webapp/RMWebApp.java | 19 +++
 .../site/markdown/yarn-service/QuickStart.md| 34 +---
 9 files changed, 89 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 640e86e..4799137 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration {
 
   public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
   + "webapp.ui2.war-file-path";
+  public static final String YARN_API_SERVICES_ENABLE = "yarn."
+  + "webapp.api-service.enable";
 
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
 RM_PREFIX + "resource-tracker.address";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e8286ef..89b020d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.yarn.service.webapp;
 
+import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,12 @@ import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 @Singleton
 @Path(CONTEXT_ROOT)
 public class ApiServer {
+
+  @Inject
+  public ApiServer(Configuration conf) {
+super();
+  }
+
   private static final Logger LOG =
   LoggerFactory.getLogger(ApiServer.class);
   private static Configuration YARN_CONFIG = new YarnConfiguration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index fb07edc..205a64d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,12 +245,6 @@
 
 
 
-  org.apache.hadoop
-  hadoop-yarn-server-resourcemanager
-  test
-
-
-
   org.apache.curator
   curator-test
   test

[57/60] [abbrv] hadoop git commit: YARN-7331. Change in few metrics in new YARN UI related to native-services. Contributed by Sunil G

[jianhe]

YARN-7331. Change in few metrics in new YARN UI related to native-services. 
Contributed by Sunil G


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a288b0ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a288b0ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a288b0ac

Branch: refs/heads/trunk
Commit: a288b0accf8e0df0375cd5ca3aafdbfb142aa8c3
Parents: ba7ed7b
Author: Jian He 
Authored: Mon Oct 30 13:16:19 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:20 2017 -0800

--
 .../main/webapp/app/adapters/yarn-servicedef.js |  2 +-
 .../app/controllers/yarn-app/components.js  |  8 ++---
 .../yarn-component-instances/info.js|  2 +-
 .../app/serializers/yarn-component-instance.js  |  2 +-
 .../app/templates/components/metrics-table.hbs  | 37 +++-
 .../templates/yarn-component-instance/info.hbs  |  6 +---
 .../src/main/webapp/config/configs.env  |  7 
 .../src/main/webapp/config/default-config.js|  3 +-
 8 files changed, 12 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
index dc5dbfd..3fb4a81 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-servicedef.js
@@ -20,7 +20,7 @@ import Ember from 'ember';
 import RESTAbstractAdapter from './restabstract';
 
 export default RESTAbstractAdapter.extend({
-  address: "dashWebAddress",
+  address: "rmWebAddress",
   restNameSpace: "dashService",
   serverName: "DASH",
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
index 947cb98..b4bac0d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-app/components.js
@@ -30,7 +30,7 @@ export default Ember.Controller.extend({
 
 cols.push({
   id: 'name',
-  headerTitle: 'Component Group',
+  headerTitle: 'Component',
   contentPath: 'name',
   cellComponentName: 'em-table-linked-cell',
   getCellContent: function(row) {
@@ -49,13 +49,9 @@ export default Ember.Controller.extend({
   contentPath: 'memory'
 }, {
   id: 'instances',
-  headerTitle: '# Components',
+  headerTitle: 'Number Of Instances',
   contentPath: 'instances',
   observePath: true
-}, {
-  id: 'createdDate',
-  headerTitle: 'Created Time',
-  contentPath: 'createdDate'
 });
 
 return ColumnDef.make(cols);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
index a676b34..0d1a4bb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-component-instances/info.js
@@ -31,7 +31,7 @@ export default Ember.Controller.extend({
 
 cols.push({
   id: 'instanceName',
-  headerTitle: 'Component Name',
+  headerTitle: 'Component Instance',
   contentPath: 'instanceName',
   cellComponentName: 'em-table-linked-cell',
   getCellContent: function(row) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a288b0ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-component-instance.js
--
diff --git 

[35/60] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
deleted file mode 100644
index f56139a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesAPI.md
+++ /dev/null
@@ -1,606 +0,0 @@
-
-
-# YARN Simplified API layer for services
-
-## Overview
-Bringing a new service on YARN today is not a simple experience. The APIs of
-existing frameworks are either too low level (native YARN), require writing
-new code (for frameworks with programmatic APIs) or writing a complex spec
-(for declarative frameworks). In addition to building critical building blocks
-inside YARN (as part of other efforts at
-[YARN-4692](https://issues.apache.org/jira/browse/YARN-4692)), there is a need 
for
-simplifying the user facing story for building services. Experience of projects
-like Apache Slider running real-life services like HBase, Storm, Accumulo,
-Solr etc, gives us some very good insights on how simplified APIs for services
-should look like.
-
-To this end, we should look at a new simple-services API layer backed by REST
-interfaces. This API can be used to create and manage the lifecycle of YARN
-services. Services here can range from simple single-component service to
-complex multi-component assemblies needing orchestration.
-[YARN-4793](https://issues.apache.org/jira/browse/YARN-4793) tracks this
-effort.
-
-This document spotlights on this specification. In most of the cases, the
-application owner will not be forced to make any changes to their applications.
-This is primarily true if the application is packaged with containerization
-technologies like docker. Irrespective of how complex the application is,
-there will be hooks provided at appropriate layers to allow pluggable and
-customizable application behavior.
-
-
-### Version information
-Version: 1.0.0
-
-### License information
-License: Apache 2.0
-License URL: http://www.apache.org/licenses/LICENSE-2.0.html
-
-### URI scheme
-Host: host.mycompany.com
-
-BasePath: /ws/v1/
-
-Schemes: HTTP
-
-### Consumes
-
-* application/json
-
-
-### Produces
-
-* application/json
-
-
-## Paths
-### Create a service
-```
-POST /services
-```
-
- Description
-
-Create a service. The request JSON is a service object with details required 
for creation. If the request is successful it returns 202 Accepted. A success 
of this API only confirms success in submission of the service creation 
request. There is no guarantee that the service will actually reach a RUNNING 
state. Resource availability and several other factors determines if the 
service will be deployed in the cluster. It is expected that clients would 
subsequently call the GET API to get details of the service and determine its 
state.
-
- Parameters
-|Type|Name|Description|Required|Schema|Default|
-|||||||
-|BodyParameter|Service|Service request object|true|Service||
-
-
- Responses
-|HTTP Code|Description|Schema|
-||||
-|202|The request to create a service is accepted|No Content|
-|400|Invalid service definition provided in the request body|No Content|
-|500|Failed to create a service|No Content|
-|default|Unexpected error|ServiceStatus|
-
-
-### (TBD) List of services running in the cluster.
-```
-GET /services
-```
-
- Description
-
-Get a list of all currently running services (response includes a minimal 
projection of the service info). For more details do a GET on a specific 
service name.
-
- Responses
-|HTTP Code|Description|Schema|
-||||
-|200|An array of services|Service array|
-|default|Unexpected error|ServiceStatus|
-
-
-### Get current version of the API server.
-```
-GET /services/version
-```
-
- Description
-
-Get current version of the API server.
-
- Responses
-|HTTP Code|Description|Schema|
-||||
-|200|Successful request|No Content|
-
-
-### Update a service or upgrade the binary version of the components of a 
running service
-```
-PUT /services/{service_name}
-```
-
- Description
-
-Update the runtime properties of a service. Currently the following operations 
are supported - update lifetime, stop/start a service. The PUT operation is 
also used to orchestrate an upgrade of the service containers to a newer 
version of their artifacts (TBD).
-
- Parameters
-|Type|Name|Description|Required|Schema|Default|
-|||||||
-|PathParameter|service_name|Service name|true|string||
-|BodyParameter|Service|The updated service definition. It can contain the 
updated lifetime of a service or the desired state (STOPPED/STARTED) 

[51/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index f22d487..df4b1df 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
 import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.Records;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 
-import static 
org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
-import static 
org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
 import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.*;
 
 public class TestServiceCLI {
+  private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
+  .class);
 
-  protected Configuration conf = new YarnConfiguration();
+  private Configuration conf = new YarnConfiguration();
   private File basedir;
-  private ServiceCLI cli;
   private SliderFileSystem fs;
+  private String basedirProp;
 
-  private void buildApp(String appDef) throws Throwable {
-String[] args =
-{ "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
-ClientArgs clientArgs = new ClientArgs(args);
-clientArgs.parse();
-cli.exec(clientArgs);
+  private void runCLI(String[] args) throws Exception {
+LOG.info("running CLI: yarn {}", Arrays.asList(args));
+ApplicationCLI cli = new ApplicationCLI();
+cli.setSysOutPrintStream(System.out);
+cli.setSysErrPrintStream(System.err);
+int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+cli.stop();
+  }
+
+  private void buildApp(String serviceName, String appDef) throws Throwable {
+String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ExampleAppJson.resourceName(appDef)};
+runCLI(args);
+  }
+
+  private void buildApp(String serviceName, String appDef, String lifetime,
+  String queue) throws Throwable {
+String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime,
+"-changeQueue", queue};
+runCLI(args);
   }
 
   @Before
   public void setup() throws Throwable {
 basedir = new File("target", "apps");
+basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
 conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
-conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
-conf.setInt(
-CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
-conf.setInt(CommonConfigurationKeysPublic.
-IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
 fs = new SliderFileSystem(conf);
 if (basedir.exists()) {
   FileUtils.deleteDirectory(basedir);
 } else {
   basedir.mkdirs();
 }
-
-// create a CLI and skip 

[47/60] [abbrv] hadoop git commit: YARN-7323. Data structure update in service REST API. Contributed by Jian He

[jianhe]

YARN-7323. Data structure update in service REST API. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68acd88d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68acd88d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68acd88d

Branch: refs/heads/trunk
Commit: 68acd88dcbfe03a0134c60b5398dfaa31ad2b786
Parents: 3c30b1a
Author: Billie Rinaldi 
Authored: Tue Oct 17 11:50:16 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 .../hadoop/yarn/service/webapp/ApiServer.java   |  9 ---
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 67 -
 .../hadoop/yarn/service/ServiceScheduler.java   |  1 +
 .../yarn/service/api/records/Component.java | 22 +-
 .../service/api/records/ComponentState.java | 30 
 .../yarn/service/api/records/Container.java | 15 ++--
 .../yarn/service/api/records/Service.java   | 76 
 .../yarn/service/api/records/ServiceState.java  |  2 +-
 .../yarn/service/client/ServiceClient.java  | 45 
 .../yarn/service/component/Component.java   | 12 
 .../component/instance/ComponentInstance.java   | 48 +
 .../yarn/service/conf/RestApiConstants.java |  4 --
 .../yarn/service/utils/ServiceApiUtil.java  | 53 ++
 .../hadoop/yarn/service/ServiceTestUtils.java   |  4 +-
 .../hadoop/yarn/service/TestServiceApiUtil.java | 74 ++-
 .../yarn/service/TestYarnNativeServices.java|  4 +-
 .../service/conf/examples/app-override.json |  8 ++-
 .../hadoop/yarn/service/conf/examples/app.json  |  8 ++-
 .../yarn/service/conf/examples/external0.json   | 15 ++--
 .../markdown/yarn-service/YarnServiceAPI.md | 22 +++---
 20 files changed, 232 insertions(+), 287 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 5773069..1bb6c93 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -242,15 +242,6 @@ public class ApiServer {
   return updateLifetime(appName, updateServiceData);
 }
 
-// flex a single component app
-if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
-.hasComponent(updateServiceData)) {
-  Component defaultComp = ServiceApiUtil
-  .createDefaultComponent(updateServiceData);
-  return updateComponent(updateServiceData.getName(), 
defaultComp.getName(),
-  defaultComp);
-}
-
 // If nothing happens consider it a no-op
 return Response.status(Status.NO_CONTENT).build();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index b9b5b3a..cc76259 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -19,28 +19,17 @@ swagger: '2.0'
 info:
   title: "YARN Simplified API layer for services"
   description: |
-Bringing a new service on YARN today is not a simple experience. The APIs 
of
-existing frameworks are either too low level (native YARN), require writing
-new code (for frameworks with programmatic APIs) or writing a complex spec
-(for declarative frameworks). In addition to building critical building 

[32/60] [abbrv] hadoop git commit: Rebased onto trunk - fix conflicts

[jianhe]

Rebased onto trunk - fix conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/307d55b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/307d55b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/307d55b3

Branch: refs/heads/trunk
Commit: 307d55b3e1ec9261c73e63286887c0b90a0c961e
Parents: 541b64c
Author: Jian He 
Authored: Mon Sep 11 17:34:30 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:15 2017 -0800

--
 .../org/apache/hadoop/yarn/service/ServiceScheduler.java | 11 +++
 1 file changed, 11 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/307d55b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index fb2fd16..7b809b9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -574,6 +574,11 @@ public class ServiceScheduler extends CompositeService {
 
 }
 
+@Override public void onContainerResourceUpdated(ContainerId containerId,
+Resource resource) {
+
+}
+
 @Override public void onGetContainerStatusError(ContainerId containerId,
 Throwable t) {
 
@@ -586,6 +591,12 @@ public class ServiceScheduler extends CompositeService {
 }
 
 @Override
+public void onUpdateContainerResourceError(ContainerId containerId,
+Throwable t) {
+
+}
+
+@Override
 public void onStopContainerError(ContainerId containerId, Throwable t) {
 
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[38/60] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He

[jianhe]

YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce74e643
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce74e643
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce74e643

Branch: refs/heads/trunk
Commit: ce74e64363abc64561263be70a923ab3e67f043f
Parents: 37c9b73
Author: Billie Rinaldi 
Authored: Thu Sep 21 10:18:42 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:16 2017 -0800

--
 .../hadoop/yarn/service/ServiceScheduler.java |  6 +-
 .../hadoop/yarn/service/client/ServiceClient.java |  3 +++
 .../params/AbstractClusterBuildingActionArgs.java |  2 +-
 .../yarn/service/client/params/Arguments.java |  1 +
 .../component/instance/ComponentInstance.java | 10 +-
 .../service/provider/AbstractProviderService.java | 18 +++---
 .../client/binding/RegistryTypeUtils.java |  3 ++-
 .../dns/ApplicationServiceRecordProcessor.java| 15 ++-
 .../server/dns/BaseServiceRecordProcessor.java|  8 
 .../registry/server/dns/TestRegistryDNS.java  | 12 
 .../runtime/DockerLinuxContainerRuntime.java  |  4 ++--
 .../src/site/markdown/YarnCommands.md |  1 +
 12 files changed, 53 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 7b809b9..ec5f3ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService {
 attemptId.getApplicationId().toString());
 serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
 PersistencePolicies.APPLICATION);
-serviceRecord.description = "Yarn Service Master";
-
-serviceRecord.addExternalEndpoint(RegistryTypeUtils
-.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
-context.clientAMService.getBindAddress()));
+serviceRecord.description = "YarnServiceMaster";
 
 // set any provided attributes
 setUserProvidedServiceRecordAttributes(service.getConfiguration(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 6890aef..a3a9fd0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService
 if (!StringUtils.isEmpty(args.getServiceName())) {
   service.setName(args.getServiceName());
 }
+if (!StringUtils.isEmpty(args.queue)) {
+  service.setQueue(args.queue);
+}
 return service;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
--
diff --git 

[20/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 000..cb7131e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,494 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler {
+  private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+  private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
+  private long allocateId;
+  private Priority priority;
+  private ServiceMetrics componentMetrics;
+  private ServiceScheduler scheduler;
+  private ServiceContext context;
+  private AMRMClientAsync amrmClient;
+  private AtomicLong instanceIdCounter = new AtomicLong();
+  private Map compInstances =
+  new ConcurrentHashMap<>();
+  // component instances to be assigned with a container
+  private List pendingInstances = new LinkedList<>();
+  private ContainerFailureTracker 

[55/60] [abbrv] hadoop git commit: YARN-7423. Improve service client loggings. Contributed by Jian He

[jianhe]

YARN-7423. Improve service client loggings. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a127f7b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a127f7b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a127f7b0

Branch: refs/heads/trunk
Commit: a127f7b0fbde3942ca4329d5463bb8fe421daf7b
Parents: a6c4bd7
Author: Billie Rinaldi 
Authored: Sat Nov 4 12:27:42 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:20 2017 -0800

--
 .../hadoop/yarn/service/ServiceMaster.java  |   4 +-
 .../yarn/service/api/records/Configuration.java |   8 +-
 .../yarn/service/client/ServiceClient.java  |  50 +-
 .../yarn/service/component/Component.java   |  10 +-
 .../component/instance/ComponentInstance.java   |   4 +-
 .../containerlaunch/AbstractLauncher.java   |   6 +-
 .../containerlaunch/ClasspathConstructor.java   |   4 +-
 .../containerlaunch/CommandLineBuilder.java |   4 +-
 .../containerlaunch/JavaCommandLineBuilder.java |   4 +-
 .../yarn/service/monitor/probe/HttpProbe.java   |   4 +-
 .../yarn/service/monitor/probe/PortProbe.java   |   4 +-
 .../provider/AbstractClientProvider.java|   8 +-
 .../provider/AbstractProviderService.java   |   4 +-
 .../yarn/service/provider/ProviderUtils.java|  10 +-
 .../registry/YarnRegistryViewForProviders.java  |   6 +-
 .../yarn/service/utils/CoreFileSystem.java  |   2 +-
 .../hadoop/yarn/service/utils/PortScanner.java  |   4 +-
 .../yarn/service/utils/ServiceApiUtil.java  |   2 +-
 .../hadoop/yarn/service/utils/ServiceUtils.java | 545 ++
 .../hadoop/yarn/service/utils/SliderUtils.java  | 564 ---
 .../yarn/service/utils/ZookeeperUtils.java  |   4 +-
 .../service/conf/TestValidateServiceNames.java  |   1 -
 22 files changed, 614 insertions(+), 638 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 2d86a5a..b0b4f06 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -39,7 +39,7 @@ import 
org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
 import org.apache.hadoop.yarn.service.monitor.ServiceMonitor;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
 import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -112,7 +112,7 @@ public class ServiceMaster extends CompositeService {
   }
 
   protected ContainerId getAMContainerId() throws BadClusterStateException {
-return ContainerId.fromString(SliderUtils.mandatoryEnvVariable(
+return ContainerId.fromString(ServiceUtils.mandatoryEnvVariable(
 ApplicationConstants.Environment.CONTAINER_ID.name()));
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a127f7b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
index dc6dff8..859c1ea 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ 

[07/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
deleted file mode 100644
index 7baa284..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
-  protected final List argumentList = new ArrayList<>(20);
-
-  /**
-   * Add an entry to the command list
-   * @param args arguments -these will be converted strings
-   */
-  public void add(Object... args) {
-for (Object arg : args) {
-  argumentList.add(arg.toString());
-}
-  }
-
-  // Get the number of arguments
-  public int size() {
-return argumentList.size();
-  }
-  
-  /**
-   * Append the output and error files to the tail of the command
-   * @param stdout out
-   * @param stderr error. Set this to null to append into stdout
-   */
-  public void addOutAndErrFiles(String stdout, String stderr) {
-Preconditions.checkNotNull(stdout, "Null output file");
-Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
-// write out the path output
-argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
-if (stderr != null) {
-  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
-   stderr);
-} else {
-  argumentList.add("2>&1");
-}
-  }
-
-  /**
-   * This just returns the command line
-   * @see #build()
-   * @return the command line
-   */
-  @Override
-  public String toString() {
-return build();
-  }
-
-  /**
-   * Build the command line
-   * @return the command line
-   */
-  public String build() {
-return SliderUtils.join(argumentList, " ");
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
deleted file mode 100644
index fcbb69b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you 

[45/60] [abbrv] hadoop git commit: YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi

[jianhe]

YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92734800
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92734800
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92734800

Branch: refs/heads/trunk
Commit: 92734800844b93bb19f3c0fca6be88d2801f7b1b
Parents: b57144a
Author: Jian He 
Authored: Fri Oct 13 10:49:38 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn|  6 ++
 .../hadoop-yarn/conf/yarn-env.sh| 12 +++
 .../hadoop-yarn/hadoop-yarn-registry/pom.xml| 73 +-
 .../dns/PrivilegedRegistryDNSStarter.java   | 80 
 .../hadoop/registry/server/dns/RegistryDNS.java | 45 +++
 .../registry/server/dns/RegistryDNSServer.java  | 57 +-
 .../src/site/markdown/YarnCommands.md   |  8 ++
 .../src/site/markdown/registry/index.md |  1 +
 .../markdown/registry/registry-configuration.md |  4 +-
 .../site/markdown/yarn-service/RegistryDNS.md   | 40 +-
 .../markdown/yarn-service/ServiceDiscovery.md   | 21 -
 11 files changed, 282 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 05d892b..727384c 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -45,6 +45,7 @@ function hadoop_usage
   hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker"
   hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server"
   hadoop_add_subcommand "queue" client "prints queue information"
+  hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
   hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" admin "admin tools"
   hadoop_add_subcommand "router" daemon "run the Router daemon"
@@ -143,6 +144,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
 queue)
   HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
 ;;
+registrydns)
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+  
HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter'
+  
HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer'
+;;
 resourcemanager)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'

http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh 
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 90a87bf..4bd1d3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -160,3 +160,15 @@
 # See ResourceManager for some examples
 #
 #export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails 
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps 
-Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
+
+###
+# Registry DNS specific parameters
+###
+# For privileged registry DNS, user to run as after dropping privileges
+# This will replace the hadoop.id.str Java property in secure mode.
+# export YARN_REGISTRYDNS_SECURE_USER=yarn
+
+# Supplemental options for privileged registry DNS
+# By default, Hadoop uses jsvc which needs to know to launch a
+# server jvm.
+# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index e083312..4e805cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -34,6 +34,21 @@
   
 
 
+  org.slf4j
+  slf4j-api
+
+
+
+  org.apache.hadoop
+  hadoop-auth
+
+
+
+  org.apache.hadoop
+  hadoop-annotations
+
+
+
   org.apache.hadoop
   hadoop-yarn-api
 
@@ -70,14 +85,68 @@
 
 
 
+  org.apache.zookeeper
+  zookeeper
+
+
+
+  org.apache.curator
+  curator-client
+
+
+

[29/60] [abbrv] hadoop git commit: Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S

[jianhe]

Rebase onto trunk - fix some issues. Contributed by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eed9e3e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eed9e3e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eed9e3e8

Branch: refs/heads/trunk
Commit: eed9e3e8f4724ca42e0140754b8fe598b6472747
Parents: e660e52
Author: Jian He 
Authored: Mon Sep 4 22:07:24 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:14 2017 -0800

--
 .../hadoop-yarn-services-api/pom.xml|  2 +-
 .../hadoop-yarn-services-core/pom.xml   |  2 +-
 .../ServiceTimelinePublisher.java   | 32 
 .../TestServiceTimelinePublisher.java   |  2 --
 .../hadoop-yarn-services/pom.xml|  2 +-
 .../src/main/webapp/app/routes/yarn-services.js |  2 +-
 6 files changed, 23 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 1077ccd..74d9681 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -19,7 +19,7 @@
   
 org.apache.hadoop
 hadoop-yarn-applications
-3.0.0-beta1-SNAPSHOT
+3.1.0-SNAPSHOT
   
   hadoop-yarn-services-api
   Apache Hadoop YARN Services API

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index d9b7adb..fb07edc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -19,7 +19,7 @@
   
 org.apache.hadoop
 hadoop-yarn-services
-3.0.0-beta1-SNAPSHOT
+3.1.0-SNAPSHOT
   
   hadoop-yarn-services-core
   jar

http://git-wip-us.apache.org/repos/asf/hadoop/blob/eed9e3e8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index 5e65ad9..9e7d8e8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -66,6 +66,7 @@ public class ServiceTimelinePublisher extends 
CompositeService {
   protected void serviceInit(org.apache.hadoop.conf.Configuration 
configuration)
   throws Exception {
 addService(timelineClient);
+super.serviceInit(configuration);
   }
 
 
@@ -95,8 +96,8 @@ public class ServiceTimelinePublisher extends 
CompositeService {
 // create info keys
 Map entityInfos = new HashMap();
 entityInfos.put(ServiceTimelineMetricsConstants.NAME, service.getName());
-entityInfos.put(ServiceTimelineMetricsConstants.STATE,
-service.getState().toString());
+//entityInfos.put(ServiceTimelineMetricsConstants.STATE,
+//service.getState().toString());
 entityInfos.put(ServiceTimelineMetricsConstants.LAUNCH_TIME,
 currentTimeMillis);
 entity.addInfo(ServiceTimelineMetricsConstants.QUICK_LINKS,
@@ -232,18 +233,23 @@ public class 

[12/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
deleted file mode 100644
index 8c968dc..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.component.ComponentEventType;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import 

[22/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
new file mode 100644
index 000..c3a2752
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -0,0 +1,892 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import 

[53/60] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi

[jianhe]

YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba7ed7b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba7ed7b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba7ed7b6

Branch: refs/heads/trunk
Commit: ba7ed7b66a48c61086ea837f39ff0f22ffe31eca
Parents: 8ec9e72
Author: Jian He 
Authored: Sat Oct 28 23:04:19 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:19 2017 -0800

--
 NOTICE.txt  |   4 -
 hadoop-project/pom.xml  |   6 -
 hadoop-yarn-project/hadoop-yarn/bin/yarn|  27 +-
 .../hadoop-yarn/conf/yarn-env.sh|  18 +-
 .../hadoop-yarn-services-core/pom.xml   |   5 -
 .../hadoop/yarn/service/ServiceMaster.java  |  21 +-
 .../hadoop/yarn/service/client/ServiceCLI.java  | 113 ---
 .../yarn/service/client/ServiceClient.java  | 187 +--
 .../client/params/AbstractActionArgs.java   | 139 
 .../client/params/AbstractArgsDelegate.java |  28 --
 .../AbstractClusterBuildingActionArgs.java  |  46 ---
 .../service/client/params/ActionBuildArgs.java  |  39 ---
 .../service/client/params/ActionClientArgs.java |  71 -
 .../service/client/params/ActionCreateArgs.java |  46 ---
 .../client/params/ActionDependencyArgs.java |  65 
 .../client/params/ActionDestroyArgs.java|  37 ---
 .../service/client/params/ActionExistsArgs.java |  49 ---
 .../service/client/params/ActionFlexArgs.java   |  50 ---
 .../service/client/params/ActionFreezeArgs.java |  56 
 .../service/client/params/ActionHelpArgs.java   |  44 ---
 .../service/client/params/ActionKeytabArgs.java |  76 -
 .../service/client/params/ActionListArgs.java   |  76 -
 .../client/params/ActionRegistryArgs.java   | 218 -
 .../client/params/ActionResolveArgs.java| 153 -
 .../client/params/ActionResourceArgs.java   |  70 
 .../service/client/params/ActionStatusArgs.java |  51 ---
 .../service/client/params/ActionThawArgs.java   |  67 
 .../service/client/params/ActionTokensArgs.java |  78 -
 .../service/client/params/ActionUpdateArgs.java |  37 ---
 .../yarn/service/client/params/ArgOps.java  | 156 -
 .../yarn/service/client/params/Arguments.java   | 103 --
 .../yarn/service/client/params/ClientArgs.java  | 193 ---
 .../yarn/service/client/params/CommonArgs.java  | 245 --
 .../client/params/ComponentArgsDelegate.java|  52 ---
 .../client/params/DontSplitArguments.java   |  34 --
 .../client/params/LaunchArgsAccessor.java   |  30 --
 .../client/params/LaunchArgsDelegate.java   |  51 ---
 .../client/params/OptionArgsDelegate.java   |  66 
 .../client/params/PathArgumentConverter.java|  34 --
 .../service/client/params/ServiceAMArgs.java|  50 ---
 .../client/params/ServiceAMCreateAction.java|  37 ---
 .../service/client/params/SliderActions.java|  79 -
 .../service/client/params/WaitArgsDelegate.java |  42 ---
 .../service/client/params/WaitTimeAccessor.java |  24 --
 .../hadoop/yarn/service/utils/SliderUtils.java  |   6 +-
 .../client/TestBuildExternalComponents.java |  19 +-
 .../yarn/service/client/TestServiceCLI.java | 125 +++-
 .../hadoop/yarn/client/api/AppAdminClient.java  | 222 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  | 318 ---
 .../hadoop/yarn/client/cli/TestYarnCLI.java | 158 ++---
 .../src/site/markdown/YarnCommands.md   |  86 ++---
 .../src/site/markdown/yarn-service/Examples.md  |  12 +-
 .../site/markdown/yarn-service/QuickStart.md|  46 ++-
 .../markdown/yarn-service/YarnServiceAPI.md |  20 +-
 54 files changed, 840 insertions(+), 3245 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index f3af2f7..bc7a26f 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -582,10 +582,6 @@ Ehcache 3.3.1,
 which has the following notices:
  * Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
 
-JCommander (https://github.com/cbeust/jcommander),
-which has the following notices:
- * Copyright 2010 Cedric Beust ced...@beust.com
-
 The binary distribution of this product bundles binaries of
 snakeyaml (https://bitbucket.org/asomov/snakeyaml),
 which has the following notices:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d18c9a8..f99e413 

[50/60] [abbrv] hadoop git commit: YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He

[jianhe]

YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2775fbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2775fbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2775fbc

Branch: refs/heads/trunk
Commit: d2775fbccaa187729d86459542210434ce4c3b8b
Parents: 501be9b
Author: Billie Rinaldi 
Authored: Wed Oct 25 13:01:01 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:19 2017 -0800

--
 .../org/apache/hadoop/registry/server/dns/RegistryDNS.java   | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2775fbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 37e8429..9097c46 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,7 +189,8 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
 
 LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
 addNIOUDP(addr, port);
-addNIOTCP(addr, port);
+//TODO Fix dns lookup over TCP
+//addNIOTCP(addr, port);
   }
 
   /**
@@ -770,6 +771,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
   byte[] response = null;
   try {
 query = new Message(in);
+LOG.info("received TCP query {}", query.getQuestion());
 response = generateReply(query, ch.socket());
 if (response == null) {
   return;
@@ -947,7 +949,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
   input.flip();
   input.get(in);
   query = new Message(in);
-  LOG.info("{}:  received query {}", remoteAddress,
+  LOG.info("{}: received UDP query {}", remoteAddress,
   query.getQuestion());
   response = generateReply(query, null);
   if (response == null) {
@@ -960,7 +962,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
 output.put(response);
 output.flip();
 
-LOG.info("{}:  sending response", remoteAddress);
+LOG.debug("{}:  sending response", remoteAddress);
 channel.send(output, remoteAddress);
   }
 } catch (Exception e) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[27/60] [abbrv] hadoop git commit: YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang

[jianhe]

YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by 
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59fe98cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59fe98cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59fe98cd

Branch: refs/heads/trunk
Commit: 59fe98cd50b34eb75c5f522aa6f3a53bd650b15e
Parents: fecf22b
Author: Jian He 
Authored: Tue Aug 29 20:37:38 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:13 2017 -0800

--
 .../java/org/apache/hadoop/yarn/service/api/records/Resource.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/59fe98cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
index cec9de9..dfdf92a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty;
 
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -103,6 +104,7 @@ public class Resource extends BaseResource implements 
Cloneable {
 this.memory = memory;
   }
 
+  @JsonIgnore
   public long getMemoryMB() {
 if (this.memory == null) {
   return 0;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[02/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
-  public static final int DEFAULT_PORT = 2181;
-
-  public static String buildConnectionString(String zkHosts, int port) {
-String zkPort = Integer.toString(port);
-//parse the hosts
-String[] hostlist = zkHosts.split(",", 0);
-String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
-return quorum;
-  }
-
-  /**
-   * Take a quorum list and split it to (trimmed) pairs
-   * @param hostPortQuorumList list of form h1:port, h2:port2,...
-   * @return a possibly empty list of values between commas. They may not be
-   * valid hostname:port pairs
-   */
-  public static List splitToPairs(String hostPortQuorumList) {
-// split an address hot
-String[] strings = StringUtils.getStrings(hostPortQuorumList);
-int len = 0;
-if (strings != null) {
-  len = strings.length;
-}
-List tuples = new ArrayList(len);
-if (strings != null) {
-  for (String s : strings) {
-tuples.add(s.trim());
-  }
-}
-return tuples;
-  }
-
-  /**
-   * Split a quorum list into a list of hostnames and ports
-   * @param hostPortQuorumList split to a list of hosts and ports
-   * @return a list of values
-   */
-  public static List splitToHostsAndPorts(String 
hostPortQuorumList) {
-// split an address hot
-String[] strings = StringUtils.getStrings(hostPortQuorumList);
-int len = 0;
-if (strings != null) {
-  len = strings.length;
-}
-List list = new ArrayList(len);
-if (strings != null) {
-  for (String s : strings) {
-
list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
-  }
-}
-return list;
-  }
-
-  /**
-   * Build up to a hosts only list
-   * @param hostAndPorts
-   * @return a list of the hosts only
-   */
-  public static String buildHostsOnlyList(List hostAndPorts) {
-StringBuilder sb = new StringBuilder();
-for (HostAndPort hostAndPort : hostAndPorts) {
-  sb.append(hostAndPort.getHostText()).append(",");
-}
-if (sb.length() > 0) {
-  sb.delete(sb.length() - 1, sb.length());
-}
-return sb.toString();
-  }
-
-  public static String buildQuorumEntry(HostAndPort hostAndPort,
-int defaultPort) {
-String s = hostAndPort.toString();
-if (hostAndPort.hasPort()) {
-  return s;
-} else {
-  return s + ":" + defaultPort;
-}
-  }
-
-  /**
-   * Build a quorum list, injecting a ":defaultPort" ref if needed on
-   * any entry without one
-   * @param hostAndPorts
-   * @param defaultPort
-   * @return
-   */
-  public static String buildQuorum(List hostAndPorts, int 
defaultPort) {
-List entries = new ArrayList(hostAndPorts.size());
-for (HostAndPort hostAndPort : hostAndPorts) {
-  entries.add(buildQuorumEntry(hostAndPort, defaultPort));
-}
-return SliderUtils.join(entries, ",", false);
-  }
-  
-  public static String convertToHostsOnlyList(String quorum) throws
-  BadConfigException {
-List 

[37/60] [abbrv] hadoop git commit: YARN-7186. Add examples in yarn-service. Contributed by Jian He

[jianhe]

YARN-7186. Add examples in yarn-service. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37c9b732
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37c9b732
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37c9b732

Branch: refs/heads/trunk
Commit: 37c9b7327d188ccad7fd36b7466a65f68ad0c899
Parents: 307d55b
Author: Billie Rinaldi 
Authored: Thu Sep 14 15:43:00 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:16 2017 -0800

--
 .../resources/assemblies/hadoop-yarn-dist.xml   |  7 
 .../examples/sleeper/sleeper.json   | 15 +++
 .../yarn/service/client/ServiceClient.java  | 26 ++--
 .../client/params/AbstractActionArgs.java   |  2 +-
 .../AbstractClusterBuildingActionArgs.java  |  2 +-
 .../service/client/params/ActionBuildArgs.java  |  7 +++-
 .../service/client/params/ActionCreateArgs.java | 12 +-
 .../service/client/params/ActionExamples.java   | 26 
 .../yarn/service/client/params/Arguments.java   |  4 +-
 .../yarn/service/client/params/ClientArgs.java  | 44 
 .../yarn/service/client/params/CommonArgs.java  | 15 +--
 .../service/client/params/SliderActions.java|  2 -
 .../src/site/markdown/YarnCommands.md   | 34 ---
 .../site/markdown/yarn-service/QuickStart.md|  9 +++-
 14 files changed, 91 insertions(+), 114 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8b3d292..83633ac 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,6 +98,13 @@
   etc/hadoop
 
 
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples
+  
/share/hadoop/${hadoop.component}/yarn-service-examples
+  
+**/*
+  
+
+
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
new file mode 100644
index 000..89ce527
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/sleeper/sleeper.json
@@ -0,0 +1,15 @@
+{
+  "name": "sleeper-service",
+  "components" :
+  [
+{
+  "name": "sleeper",
+  "number_of_containers": 2,
+  "launch_command": "sleep 90",
+  "resource": {
+"cpus": 1,
+"memory": "256"
+  }
+}
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/37c9b732/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index bfddc44..6890aef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import 

[08/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
deleted file mode 100644
index 982a114..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.Date;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
-import static 
org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
-import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
-import static 
org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*;
-import static 
org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*;
-
-public class ComponentInstance implements EventHandler,
-Comparable {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(ComponentInstance.class);
-
-  private  StateMachine stateMachine;
-  private Component component;
-  private final ReadLock readLock;
-  private final WriteLock writeLock;
-
-  private ComponentInstanceId compInstanceId = null;
-  private Path compInstanceDir;
-  private Container container;
-  private YarnRegistryViewForProviders yarnRegistryOperations;
-  private FileSystem fs;
-  private boolean timelineServiceEnabled = false;
-  private 

[39/60] [abbrv] hadoop git commit: YARN-7201. Added an apache httpd example YARN service. Contributed by Billie Rinaldi

[jianhe]

YARN-7201. Added an apache httpd example YARN service. Contributed by Billie 
Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c70e5df1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c70e5df1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c70e5df1

Branch: refs/heads/trunk
Commit: c70e5df100ba800f7170f44f63d388ae056cb74d
Parents: ce74e64
Author: Jian He 
Authored: Mon Sep 25 16:36:43 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:16 2017 -0800

--
 .../examples/httpd-no-dns/httpd-no-dns.json |  62 
 .../httpd-no-dns/httpd-proxy-no-dns.conf|  24 +++
 .../examples/httpd/httpd-proxy.conf |  24 +++
 .../examples/httpd/httpd.json   |  55 +++
 .../yarn/service/api/ServiceApiConstants.java   |   5 +
 .../yarn/service/component/Component.java   |  31 
 .../yarn/service/provider/ProviderUtils.java|   1 +
 .../hadoop/registry/server/dns/RegistryDNS.java |   4 +
 .../src/site/markdown/yarn-service/Examples.md  | 159 +++
 .../src/site/markdown/yarn-service/Overview.md  |   3 +-
 10 files changed, 367 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
new file mode 100644
index 000..6b35538
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-no-dns.json
@@ -0,0 +1,62 @@
+{
+  "name": "httpd-service-no-dns",
+  "lifetime": "3600",
+  "components": [
+{
+  "name": "httpd",
+  "number_of_containers": 2,
+  "artifact": {
+"id": "centos/httpd-24-centos7:latest",
+"type": "DOCKER"
+  },
+  "launch_command": "/usr/bin/run-httpd",
+  "resource": {
+"cpus": 1,
+"memory": "1024"
+  },
+  "readiness_check": {
+"type": "HTTP",
+"props": {
+  "url": "http://${THIS_HOST}:8080;
+}
+  },
+  "configuration": {
+"files": [
+  {
+"type": "ENV",
+"dest_file": "/var/www/html/index.html",
+"props": {
+  "content": 
"TitleHello from 
${COMPONENT_INSTANCE_NAME}!"
+}
+  }
+]
+  }
+},
+{
+  "name": "httpd-proxy",
+  "number_of_containers": 1,
+  "dependencies": [ "httpd" ],
+  "artifact": {
+"id": "centos/httpd-24-centos7:latest",
+"type": "DOCKER"
+  },
+  "launch_command": "/usr/bin/run-httpd",
+  "resource": {
+"cpus": 1,
+"memory": "1024"
+  },
+  "configuration": {
+"files": [
+  {
+"type": "TEMPLATE",
+"dest_file": "/etc/httpd/conf.d/httpd-proxy.conf",
+"src_file": "httpd-proxy-no-dns.conf"
+  }
+]
+  }
+}
+  ],
+  "quicklinks": {
+"Apache HTTP Server": 
"http://httpd-proxy-0.${SERVICE_NAME}.${USER}.${DOMAIN}:8080;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c70e5df1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
new file mode 100644
index 000..9894e64
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/examples/httpd-no-dns/httpd-proxy-no-dns.conf
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# 

[19/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 000..7baa284
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+  protected final List argumentList = new ArrayList<>(20);
+
+  /**
+   * Add an entry to the command list
+   * @param args arguments -these will be converted strings
+   */
+  public void add(Object... args) {
+for (Object arg : args) {
+  argumentList.add(arg.toString());
+}
+  }
+
+  // Get the number of arguments
+  public int size() {
+return argumentList.size();
+  }
+  
+  /**
+   * Append the output and error files to the tail of the command
+   * @param stdout out
+   * @param stderr error. Set this to null to append into stdout
+   */
+  public void addOutAndErrFiles(String stdout, String stderr) {
+Preconditions.checkNotNull(stdout, "Null output file");
+Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+// write out the path output
+argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+if (stderr != null) {
+  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
+   stderr);
+} else {
+  argumentList.add("2>&1");
+}
+  }
+
+  /**
+   * This just returns the command line
+   * @see #build()
+   * @return the command line
+   */
+  @Override
+  public String toString() {
+return build();
+  }
+
+  /**
+   * Build the command line
+   * @return the command line
+   */
+  public String build() {
+return SliderUtils.join(argumentList, " ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 000..0e51a62
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 

[16/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
new file mode 100644
index 000..7b22e3e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param 
+ */
+public class JsonSerDeser {
+
+  private static final Logger log = 
LoggerFactory.getLogger(JsonSerDeser.class);
+  private static final String UTF_8 = "UTF-8";
+
+  private final Class classType;
+  private final ObjectMapper mapper;
+
+  /**
+   * Create an instance bound to a specific type
+   * @param classType class type
+   */
+  public JsonSerDeser(Class classType) {
+this.classType = classType;
+this.mapper = new ObjectMapper();
+mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, 
false);
+  }
+
+  public JsonSerDeser(Class classType, PropertyNamingStrategy 
namingStrategy) {
+this(classType);
+mapper.setPropertyNamingStrategy(namingStrategy);
+  }
+
+  /**
+   * Convert from JSON
+   * @param json input
+   * @return the parsed JSON
+   * @throws IOException IO
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public T fromJson(String json)
+throws IOException, JsonParseException, JsonMappingException {
+try {
+  return mapper.readValue(json, classType);
+} catch (IOException e) {
+  log.error("Exception while parsing json : " + e + "\n" + json, e);
+  throw e;
+}
+  }
+
+  /**
+   * Convert from a JSON file
+   * @param jsonFile input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public T fromFile(File jsonFile)
+throws IOException, JsonParseException, JsonMappingException {
+File absoluteFile = jsonFile.getAbsoluteFile();
+try {
+  return mapper.readValue(absoluteFile, classType);
+} catch (IOException e) {
+  log.error("Exception while parsing json file {}", absoluteFile, e);
+  throw e;
+}
+  }
+
+  /**
+   * Convert from a JSON file
+   * @param resource input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+ public T fromResource(String resource)
+throws IOException, JsonParseException, JsonMappingException {
+

[13/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 000..b742553
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+  private TimelineV2Client timelineClient;
+  private Configuration config;
+  private ServiceTimelinePublisher serviceTimelinePublisher;
+  private static String SERVICE_NAME = "HBASE";
+  private static String SERVICEID = "application_1490093646524_0005";
+  private static String ARTIFACTID = "ARTIFACTID";
+  private static String COMPONENT_NAME = "DEFAULT";
+  private static String CONTAINER_ID =
+  "container_e02_1490093646524_0005_01_01";
+  private static String CONTAINER_IP =
+  "localhost";
+  private static String CONTAINER_HOSTNAME =
+  "cnl124-localhost.site";
+  private static String CONTAINER_BAREHOST =
+  "localhost.com";
+
+  @Before
+  public void setUp() throws Exception {
+config = new Configuration();
+config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+timelineClient =
+new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
+serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+timelineClient.init(config);
+serviceTimelinePublisher.init(config);
+timelineClient.start();
+serviceTimelinePublisher.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+if 

[34/60] [abbrv] hadoop git commit: YARN-7165. Miscellaneous fixes in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/541b64c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
new file mode 100644
index 000..9022268
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/YarnServiceAPI.md
@@ -0,0 +1,592 @@
+
+
+# YARN Service API 
+
+Bringing a new service on YARN today is not a simple experience. The APIs of 
existing 
+frameworks are either too low level (native YARN), require writing new code 
(for frameworks with programmatic APIs)
+or writing a complex spec (for declarative frameworks).
+
+This simplified REST API can be used to create and manage the lifecycle of 
YARN services. 
+In most cases, the application owner will not be forced to make any changes to 
their applications. 
+This is primarily true if the application is packaged with containerization 
technologies like Docker.
+
+This document describes the API specifications (aka. YarnFile) for 
deploying/managing
+containerized services on YARN. The same JSON spec can be used for both REST 
API
+and CLI to manage the services. 
+
+
+### Version information
+Version: 1.0.0
+
+### License information
+License: Apache 2.0
+License URL: http://www.apache.org/licenses/LICENSE-2.0.html
+
+### URI scheme
+Host: host.mycompany.com
+
+Port: 9191(default)
+
+Schemes: HTTP
+
+### Consumes
+
+* application/json
+
+
+### Produces
+
+* application/json
+
+
+## Paths
+### Create a service
+```
+POST /ws/v1/services
+```
+
+ Description
+
+Create a service. The request JSON is a service object with details required 
for creation. If the request is successful it returns 202 Accepted. A success 
of this API only confirms success in submission of the service creation 
request. There is no guarantee that the service will actually reach a RUNNING 
state. Resource availability and several other factors determines if the 
service will be deployed in the cluster. It is expected that clients would 
subsequently call the GET API to get details of the service and determine its 
state.
+
+ Parameters
+|Type|Name|Description|Required|Schema|Default|
+|||||||
+|BodyParameter|Service|Service request object|true|Service||
+
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|202|The request to create a service is accepted|No Content|
+|400|Invalid service definition provided in the request body|No Content|
+|500|Failed to create a service|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### (TBD) List of services running in the cluster.
+```
+GET /ws/v1/services
+```
+
+ Description
+
+Get a list of all currently running services (response includes a minimal 
projection of the service info). For more details do a GET on a specific 
service name.
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|200|An array of services|Service array|
+|default|Unexpected error|ServiceStatus|
+
+
+### Get current version of the API server.
+```
+GET /ws/v1/services/version
+```
+
+ Description
+
+Get current version of the API server.
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|200|Successful request|No Content|
+
+
+### Update a service or upgrade the binary version of the components of a 
running service
+```
+PUT /ws/v1/services/{service_name}
+```
+
+ Description
+
+Update the runtime properties of a service. Currently the following operations 
are supported - update lifetime, stop/start a service. The PUT operation is 
also used to orchestrate an upgrade of the service containers to a newer 
version of their artifacts (TBD).
+
+ Parameters
+|Type|Name|Description|Required|Schema|Default|
+|||||||
+|PathParameter|service_name|Service name|true|string||
+|BodyParameter|Service|The updated service definition. It can contain the 
updated lifetime of a service or the desired state (STOPPED/STARTED) of a 
service to initiate a start/stop operation against the specified 
service|true|Service||
+
+
+ Responses
+|HTTP Code|Description|Schema|
+||||
+|204|Update or upgrade was successful|No Content|
+|404|Service does not exist|No Content|
+|default|Unexpected error|ServiceStatus|
+
+
+### Destroy a service
+```
+DELETE /ws/v1/services/{service_name}
+```
+
+ Description
+
+Destroy a service and release all resources. This API might have to return 
JSON data providing location of logs (TBD), etc.
+
+ Parameters
+|Type|Name|Description|Required|Schema|Default|
+|||||||
+|PathParameter|service_name|Service name|true|string||
+
+
+ Responses
+|HTTP Code|Description|Schema|
+||||

[56/60] [abbrv] hadoop git commit: YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. Contributed by Chandni Singh

[jianhe]

YARN-7371. Added allocateRequestId in NMContainerStatus for recovery. 
Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a55d0738
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a55d0738
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a55d0738

Branch: refs/heads/trunk
Commit: a55d0738f1eba225c45578bbdabd0e5a6d6f5392
Parents: a127f7b
Author: Jian He 
Authored: Sat Nov 4 23:15:21 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:20 2017 -0800

--
 .../hadoop/yarn/service/ServiceTestUtils.java   |  10 +-
 .../yarn/service/TestYarnNativeServices.java| 105 ---
 .../yarn/security/ContainerTokenIdentifier.java |  20 +++-
 .../src/main/proto/yarn_security_token.proto|   1 +
 .../api/protocolrecords/NMContainerStatus.java  |  18 +++-
 .../impl/pb/NMContainerStatusPBImpl.java|  15 ++-
 .../OpportunisticContainerAllocator.java|   2 +-
 .../yarn_server_common_service_protos.proto |   1 +
 .../containermanager/ContainerManagerImpl.java  |   5 +-
 .../container/ContainerImpl.java|   8 +-
 .../scheduler/AbstractYarnScheduler.java|   1 +
 .../scheduler/SchedulerApplicationAttempt.java  |   3 +-
 .../security/RMContainerTokenSecretManager.java |  36 ++-
 .../server/resourcemanager/TestRMRestart.java   |   2 +-
 .../TestResourceTrackerService.java |   6 +-
 .../capacity/TestContainerAllocation.java   |   5 +-
 16 files changed, 177 insertions(+), 61 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 19a5177..cf32880 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -131,12 +131,16 @@ public class ServiceTestUtils {
 return fs;
   }
 
+  protected MiniYARNCluster getYarnCluster() {
+return yarnCluster;
+  }
+
   protected void setupInternal(int numNodeManager)
   throws Exception {
 LOG.info("Starting up YARN cluster");
-//Logger rootLogger = LogManager.getRootLogger();
-//rootLogger.setLevel(Level.DEBUG);
-setConf(new YarnConfiguration());
+if (conf == null) {
+  setConf(new YarnConfiguration());
+}
 conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
 // reduce the teardown waiting time
 conf.setLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT, 1000);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a55d0738/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 64fcf57..f98d90a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -18,12 +18,13 @@
 
 package org.apache.hadoop.yarn.service;
 
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import 

[58/60] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
index 17d38cb..0f0156a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/metrics-table.hbs
@@ -20,7 +20,7 @@
   
 
   
-{{type}} Metrics: Detailed Information
+{{type}} Metrics:
   
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
index 5a94575..ef517d0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instance/info.hbs
@@ -20,7 +20,7 @@
   {{#if model.container}}
   
 
-  Component Information
+  Component Instance Information
 
 
   
@@ -30,7 +30,7 @@
 {{check-availability model.container.instanceName}}
   
   
-Component Group
+Component
 {{check-availability model.container.component}}
   
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
index 0b642b0..fb4dba2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-component-instances/info.hbs
@@ -18,7 +18,7 @@
 
 
   
-Active Components: {{model.componentName}}
+Component: {{model.componentName}}
 {{em-table columns=tableColumns rows=model.instances}}
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[44/60] [abbrv] hadoop git commit: YARN-6744. Recover component information on YARN native services AM restart. Contributed by Billie Rinaldi

[jianhe]

YARN-6744. Recover component information on YARN native services AM restart. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a7ef1b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a7ef1b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a7ef1b

Branch: refs/heads/trunk
Commit: b8a7ef1b64392094562e7782e0fd092934724ad2
Parents: c723021
Author: Jian He 
Authored: Wed Oct 11 21:05:06 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 .../hadoop-yarn-services-core/pom.xml   |  5 ++
 .../hadoop/yarn/service/ServiceScheduler.java   | 94 ++--
 .../yarn/service/component/Component.java   | 76 +---
 .../service/component/ComponentEventType.java   |  1 +
 .../component/instance/ComponentInstance.java   | 27 +-
 .../yarn/service/provider/ProviderUtils.java| 15 ++--
 .../registry/YarnRegistryViewForProviders.java  | 52 ---
 7 files changed, 211 insertions(+), 59 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index 205a64d..851f73b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -159,6 +159,11 @@
 
 
   org.apache.hadoop
+  hadoop-yarn-server-common
+
+
+
+  org.apache.hadoop
   hadoop-common
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a7ef1b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index ec5f3ed..f3824df 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.registry.client.binding.RegistryUtils;
 import org.apache.hadoop.registry.client.types.ServiceRecord;
 import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
@@ -237,11 +237,6 @@ public class ServiceScheduler extends CompositeService {
   serviceTimelinePublisher
   .serviceAttemptUnregistered(context, diagnostics.toString());
 }
-// Cleanup each component instance. no need to release containers as
-// they will be automatically released by RM
-for (ComponentInstance instance : liveInstances.values()) {
-  instance.cleanupRegistryAndCompHdfsDir();
-}
 String msg = diagnostics.toString()
 + "Navigate to the failed component for more details.";
 amRMClient
@@ -266,11 +261,67 @@ public class ServiceScheduler extends CompositeService {
 }
 registerServiceInstance(context.attemptId, app);
 
-//TODO handle containers recover
-  }
+// recover components based on containers sent from RM
+recoverComponents(response);
 
-  private void recover() {
+for (Component component : componentsById.values()) {
+  // Trigger initial evaluation of components
+  if (component.areDependenciesReady()) {
+LOG.info("Triggering initial evaluation of component {}",
+component.getName());
+ComponentEvent event = new 

[23/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 000..c64b1b5
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+  private Integer code = null;
+  private String message = null;
+  private String fields = null;
+
+  /**
+   **/
+  public Error code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  /**
+   **/
+  public Error message(String message) {
+this.message = message;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("message")
+  public String getMessage() {
+return message;
+  }
+
+  public void setMessage(String message) {
+this.message = message;
+  }
+
+  /**
+   **/
+  public Error fields(String fields) {
+this.fields = fields;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("fields")
+  public String getFields() {
+return fields;
+  }
+
+  public void setFields(String fields) {
+this.fields = fields;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+Error error = (Error) o;
+return Objects.equals(this.code, error.code)
+&& Objects.equals(this.message, error.message)
+&& Objects.equals(this.fields, error.fields);
+  }
+
+  @Override
+  public int hashCode() {
+return Objects.hash(code, message, fields);
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder();
+sb.append("class Error {\n");
+
+sb.append("code: ").append(toIndentedString(code)).append("\n");
+sb.append("message: ").append(toIndentedString(message)).append("\n");
+sb.append("fields: ").append(toIndentedString(fields)).append("\n");
+sb.append("}");
+return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+if (o == null) {
+  return "null";
+}
+return o.toString().replace("\n", "\n");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
--
diff --git 

[03/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
deleted file mode 100644
index 405f690..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- * 
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
-  public String applicationId;
-  public String applicationAttemptId;
-  public String name;
-  public String applicationType;
-  public String user;
-  public String queue;
-  public String host;
-  public Integer rpcPort;
-  public String state;
-  public String diagnostics;
-  public String url;
-  /**
-   * This value is non-null only when a report is generated from a submission 
context.
-   * The YARN {@link ApplicationReport} structure does not propagate this value
-   * from the RM.
-   */
-  public Long submitTime;
-  public Long startTime;
-  public Long finishTime;
-  public String finalStatus;
-  public String origTrackingUrl;
-  public Float progress;
-  
-  public SerializedApplicationReport() {
-  }
-  
-  public SerializedApplicationReport(ApplicationReport report) {
-this.applicationId = report.getApplicationId().toString();
-ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
-this.applicationAttemptId = attemptId != null ? attemptId.toString() : 
"N/A";
-this.name = report.getName();
-this.applicationType = report.getApplicationType();
-this.user = report.getUser();
-this.queue = report.getQueue();
-this.host = report.getHost();
-this.rpcPort = report.getRpcPort();
-this.state = report.getYarnApplicationState().toString();
-this.diagnostics = report.getDiagnostics();
-this.startTime = report.getStartTime();
-this.finishTime = report.getFinishTime();
-FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
-this.finalStatus = appStatus == null ? "" : appStatus.toString();
-this.progress = report.getProgress();
-this.url = report.getTrackingUrl();
-this.origTrackingUrl= report.getOriginalTrackingUrl();
-  }
-
-  @Override
-  public String toString() {
-try {
-  return ApplicationReportSerDeser.toString(this);
-} catch (IOException e) {
-  return super.toString();
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
--
diff --git 

[11/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
deleted file mode 100644
index e10305a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.util.Locale;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ConfigFormat {
-
-  JSON("json"),
-  PROPERTIES("properties"),
-  XML("xml"),
-  HADOOP_XML("hadoop_xml"),
-  ENV("env"),
-  TEMPLATE("template"),
-  YAML("yaml"),
-  ;
-  ConfigFormat(String suffix) {
-this.suffix = suffix;
-  }
-
-  private final String suffix;
-
-  public String getSuffix() {
-return suffix;
-  }
-
-
-  @Override
-  public String toString() {
-return suffix;
-  }
-
-  /**
-   * Get a matching format or null
-   * @param type
-   * @return the format
-   */
-  public static ConfigFormat resolve(String type) {
-for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
-return format;
-  }
-}
-return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
deleted file mode 100644
index 0ac508b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import 

[31/60] [abbrv] hadoop git commit: YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha

[jianhe]

YARN-7073. Yarn native services rest API documentation. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7dd4490a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7dd4490a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7dd4490a

Branch: refs/heads/trunk
Commit: 7dd4490a2bd2b682a5f1a8605e6f6c053037
Parents: bd96c4c
Author: Jian He 
Authored: Thu Aug 31 17:17:22 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:14 2017 -0800

--
 hadoop-project/src/site/site.xml|   1 +
 .../hadoop/yarn/service/webapp/ApiServer.java   |  47 +-
 .../definition/YARN-Services-Examples.md| 245 
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 128 ++--
 .../native-services/NativeServicesAPI.md| 606 +++
 5 files changed, 973 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 8cb57c8..ad893a2 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -157,6 +157,7 @@
 
 
   
+  
   
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7dd4490a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index f55e3f1..e8286ef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -24,12 +24,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import org.apache.hadoop.yarn.service.api.records.ServiceStatus;
 import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,6 +48,7 @@ import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 
+import static org.apache.hadoop.yarn.service.api.records.ServiceState.ACCEPTED;
 import static org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 
 /**
@@ -76,11 +76,11 @@ public class ApiServer {
   @GET
   @Path(VERSION)
   @Consumes({ MediaType.APPLICATION_JSON })
-  @Produces({ MediaType.APPLICATION_JSON, MediaType.TEXT_PLAIN })
+  @Produces({ MediaType.APPLICATION_JSON })
   public Response getVersion() {
 String version = VersionInfo.getBuildVersion();
 LOG.info(version);
-return Response.ok(version).build();
+return Response.ok("{ \"hadoop_version\": \"" + version + "\"}").build();
   }
 
   @POST
@@ -94,11 +94,11 @@ public class ApiServer {
   ApplicationId applicationId = SERVICE_CLIENT.actionCreate(service);
   LOG.info("Successfully created service " + service.getName()
   + " applicationId = " + applicationId);
-  serviceStatus.setState(ServiceState.ACCEPTED);
+  serviceStatus.setState(ACCEPTED);
   serviceStatus.setUri(
   CONTEXT_ROOT + SERVICE_ROOT_PATH + "/" + service
   .getName());
-  return Response.status(Status.CREATED).entity(serviceStatus).build();
+  return Response.status(Status.ACCEPTED).entity(serviceStatus).build();
 } catch (IllegalArgumentException e) {
   serviceStatus.setDiagnostics(e.getMessage());
   return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
@@ -182,16 +182,16 @@ public class ApiServer {
   + ": Invalid number of containers specified " + component
   .getNumberOfContainers()).build();
 }
+ServiceStatus status = new 

[25/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 88f74ef..17f8c95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -22,7 +22,7 @@ info:
 Bringing a new service on YARN today is not a simple experience. The APIs 
of existing frameworks are either too low level (native YARN), require writing 
new code (for frameworks with programmatic APIs) or writing a complex spec (for 
declarative frameworks). In addition to building critical building blocks 
inside YARN (as part of other efforts at 
link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a 
need for simplifying the user facing story for building services. Experience of 
projects like Apache Slider running real-life services like HBase, Storm, 
Accumulo, Solr etc, gives us some very good insights on how simplified APIs for 
services should look like.
 
 
-To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component apps to 
complex multi-component assemblies needing orchestration.
+To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component service to 
complex multi-component assemblies needing orchestration.
 
 
 We should also look at making this a unified REST based entry point for 
other important features like resource-profile management 
(link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), 
package-definitions' lifecycle-management and service-discovery 
(link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]).
 We also need to flesh out its relation to our present much lower level REST 
APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN 
for application-submission and management.
@@ -41,177 +41,177 @@ schemes:
   - http
   - https
 # will be prefixed to all paths
-basePath: /services/v1/
+basePath: /ws/v1/
 consumes:
   - application/json
 produces:
   - application/json
 paths:
-  /applications:
+  /services:
 get:
-  summary: List of applications/services running in the cluster
-  description: Get a list of all currently running applications (response 
includes a minimal projection of the application info). For more details do a 
GET on a specific application name.
+  summary: List of services running in the cluster
+  description: Get a list of all currently running services (response 
includes a minimal projection of the service info). For more details do a GET 
on a specific service name.
   responses:
 200:
-  description: An array of applications
+  description: An array of services
   schema:
 type: array
 items:
-  $ref: '#/definitions/Application'
+  $ref: '#/definitions/Service'
 default:
   description: Unexpected error
   schema:
-$ref: '#/definitions/ApplicationStatus'
+$ref: '#/definitions/ServiceStatus'
 post:
-  summary: Create an application/service
-  description: Create an application. The request JSON is an Application 
object with details required for creation. If the request is successful it 
returns 202 Accepted. A success of this API only confirms success in submission 
of the application creation request. There is no guarantee that the application 
will actually reach a RUNNING state. Resource availability and several other 
factors determines if the application will be deployed in the cluster. It is 
expected that clients would subsequently call the GET API to get details of the 
application and determine its state.
+  summary: Create a service
+  description: Create a service. The request JSON is a service object with 
details required for creation. If the request is successful it returns 202 
Accepted. A success of this API only 

[26/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

YARN-7091. Rename application to service in yarn-native-services. Contributed 
by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40ab068e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40ab068e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40ab068e

Branch: refs/heads/trunk
Commit: 40ab068eabe50875449d601471619a4e82fff86d
Parents: 2ba38ce
Author: Billie Rinaldi 
Authored: Mon Aug 28 09:59:55 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:13 2017 -0800

--
 .../resources/assemblies/hadoop-yarn-dist.xml   |8 +-
 .../assemblies/hadoop-yarn-services-dist.xml|2 +-
 hadoop-project/pom.xml  |2 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   14 +-
 .../hadoop-yarn/conf/yarn-env.sh|2 +-
 .../hadoop-yarn-services-api/pom.xml|2 +-
 .../hadoop/yarn/service/webapp/ApiServer.java   |  274 +++
 .../yarn/service/webapp/ApiServerWebApp.java|  159 ++
 .../service/webapp/ApplicationApiService.java   |  275 ---
 .../service/webapp/ApplicationApiWebApp.java|  123 --
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  185 +-
 .../src/main/scripts/run_rest_service.sh|   28 -
 .../dev-support/findbugs-exclude.xml|   48 +
 .../conf/yarnservice-log4j.properties   |   62 +
 .../hadoop-yarn-services-core/pom.xml   |  408 +
 .../hadoop/yarn/service/ClientAMProtocol.java   |   40 +
 .../hadoop/yarn/service/ClientAMService.java|  132 ++
 .../yarn/service/ContainerFailureTracker.java   |   89 +
 .../hadoop/yarn/service/ServiceContext.java |   41 +
 .../hadoop/yarn/service/ServiceMaster.java  |  156 ++
 .../hadoop/yarn/service/ServiceMetrics.java |   98 +
 .../hadoop/yarn/service/ServiceScheduler.java   |  654 +++
 .../yarn/service/api/ServiceApiConstants.java   |   69 +
 .../yarn/service/api/records/Artifact.java  |  160 ++
 .../yarn/service/api/records/BaseResource.java  |   52 +
 .../yarn/service/api/records/Component.java |  412 +
 .../yarn/service/api/records/ConfigFile.java|  225 +++
 .../yarn/service/api/records/ConfigFormat.java  |   67 +
 .../yarn/service/api/records/Configuration.java |  225 +++
 .../yarn/service/api/records/Container.java |  297 +++
 .../service/api/records/ContainerState.java |   30 +
 .../hadoop/yarn/service/api/records/Error.java  |  129 ++
 .../service/api/records/PlacementPolicy.java|  102 ++
 .../service/api/records/ReadinessCheck.java |  175 ++
 .../yarn/service/api/records/Resource.java  |  159 ++
 .../yarn/service/api/records/Service.java   |  466 +
 .../yarn/service/api/records/ServiceState.java  |   33 +
 .../yarn/service/api/records/ServiceStatus.java |  148 ++
 .../yarn/service/client/ClientAMProxy.java  |   57 +
 .../hadoop/yarn/service/client/ServiceCLI.java  |  112 ++
 .../yarn/service/client/ServiceClient.java  |  892 +
 .../client/params/AbstractActionArgs.java   |  158 ++
 .../client/params/AbstractArgsDelegate.java |   28 +
 .../AbstractClusterBuildingActionArgs.java  |   58 +
 .../service/client/params/ActionBuildArgs.java  |   31 +
 .../service/client/params/ActionClientArgs.java |   71 +
 .../service/client/params/ActionCreateArgs.java |   33 +
 .../client/params/ActionDependencyArgs.java |   65 +
 .../client/params/ActionDestroyArgs.java|   37 +
 .../service/client/params/ActionExistsArgs.java |   49 +
 .../service/client/params/ActionFlexArgs.java   |   50 +
 .../service/client/params/ActionFreezeArgs.java |   56 +
 .../service/client/params/ActionHelpArgs.java   |   44 +
 .../service/client/params/ActionKDiagArgs.java  |   76 +
 .../service/client/params/ActionKeytabArgs.java |   76 +
 .../service/client/params/ActionListArgs.java   |   76 +
 .../client/params/ActionRegistryArgs.java   |  218 +++
 .../client/params/ActionResolveArgs.java|  153 ++
 .../client/params/ActionResourceArgs.java   |   70 +
 .../service/client/params/ActionStatusArgs.java |   51 +
 .../service/client/params/ActionThawArgs.java   |   67 +
 .../service/client/params/ActionTokensArgs.java |   78 +
 .../service/client/params/ActionUpdateArgs.java |   32 +
 .../yarn/service/client/params/ArgOps.java  |  156 ++
 .../yarn/service/client/params/Arguments.java   |  103 ++
 .../yarn/service/client/params/ClientArgs.java  |  252 +++
 .../yarn/service/client/params/CommonArgs.java  |  282 +++
 .../client/params/ComponentArgsDelegate.java|   52 +
 .../client/params/DontSplitArguments.java   |   34 +
 .../client/params/LaunchArgsAccessor.java   |   30 +
 .../client/params/LaunchArgsDelegate.java   |   51 +
 .../client/params/OptionArgsDelegate.java   |   66 +
 .../client/params/PathArgumentConverter.java|   34 +
 

[41/60] [abbrv] hadoop git commit: YARN-7191. Improve yarn-service documentation. Contributed by Jian He

[jianhe]

YARN-7191. Improve yarn-service documentation. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88512097
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88512097
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88512097

Branch: refs/heads/trunk
Commit: 8851209788a4e110e899dd28f628657a21981db5
Parents: c70e5df
Author: Billie Rinaldi 
Authored: Wed Sep 27 15:08:33 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:17 2017 -0800

--
 .../src/site/markdown/yarn-service/Concepts.md  |  47 +---
 .../src/site/markdown/yarn-service/Overview.md  |   3 +-
 .../site/markdown/yarn-service/QuickStart.md|  34 +--
 .../site/markdown/yarn-service/RegistryDNS.md   | 166 +
 .../markdown/yarn-service/ServiceDiscovery.md   | 235 ---
 5 files changed, 286 insertions(+), 199 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
index 7b62c36..e567d03 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -22,6 +22,8 @@ It also does all the heavy lifting work such as resolving the 
service definition
 failed containers, monitoring components' healthiness and readiness, ensuring 
dependency start order across components, flexing up/down components, 
 upgrading components etc. The end goal of the framework is to make sure the 
service is up and running as the state that user desired.
 
+In addition, it leverages a lot of features in YARN core to accomplish 
scheduling constraints, such as
+affinity and anti-affinity scheduling, log aggregation for services, 
automatically restart a container if it fails, and do in-place upgrade of a 
container.
 
 ### A Restful API-Server for deploying/managing services on YARN
 A restful API server is developed to allow users to deploy/manage their 
services on YARN via a simple JSON spec. This avoids users
@@ -34,44 +36,11 @@ support HA, distribute the load etc.
 
 ### Service Discovery
 A DNS server is implemented to enable discovering services on YARN via the 
standard mechanism: DNS lookup.
-The DNS server essentially exposes the information in YARN service registry by 
translating them into DNS records such as A record and SRV record.
-Clients can discover the IPs of containers via standard DNS lookup.
-The previous read mechanisms of YARN Service Registry were limited to a 
registry specific (java) API and a REST interface and are difficult
-to wireup existing clients and services. The DNS based service discovery 
eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md) 
-for more details.
-
-### Scheduling
-
-A host of scheduling features are being developed to support long running 
services.
-
-* Affinity and anti-affinity scheduling across containers 
([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)).
-* Container resizing 
([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
-* Special handling of container preemption/reservation for services 
-
-### Container auto-restarts
-
-[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a 
retry-policy to let NM re-launch a service container when it fails.
-The service REST API provides users a way to enable NodeManager to 
automatically restart the container if it fails.
-The advantage is that it avoids the entire cycle of releasing the failed 
containers, re-asking new containers, re-do resource localizations and so on, 
which
-greatly minimizes container downtime.
 
+The framework posts container information such as hostname and ip into the 
[YARN service registry](../registry/index.md). And the DNS server essentially 
exposes the
+information in YARN service registry by translating them into DNS records such 
as A record and SRV record.
+Clients can then discover the IPs of containers via standard DNS lookup.
 
-### Container in-place upgrade
-
-[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support 
upgrading containers in-place, that is, without losing the container 
allocations.
-It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade 
their containers via a simple API call.
-Under the hood, NodeManager does below steps:
-* Downloading the new resources such as jars, docker container 

[60/60] [abbrv] hadoop git commit: Merge branch 'yarn-native-services' into trunk

[jianhe]

Merge branch 'yarn-native-services' into trunk


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbc632d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbc632d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbc632d9

Branch: refs/heads/trunk
Commit: cbc632d9abf08c56a7fc02be51b2718af30bad28
Parents: dcd99c4 a55d073
Author: Jian He 
Authored: Mon Nov 6 14:02:19 2017 -0800
Committer: Jian He 
Committed: Mon Nov 6 14:02:19 2017 -0800

--
 LICENSE.txt |1 +
 NOTICE.txt  |   10 +
 .../resources/assemblies/hadoop-yarn-dist.xml   |   25 +
 hadoop-project/pom.xml  |   44 +-
 hadoop-project/src/site/site.xml|   10 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   18 +-
 .../hadoop-yarn/conf/yarn-env.sh|   18 +
 .../dev-support/findbugs-exclude.xml|   16 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |2 +
 .../dev-support/findbugs-exclude.xml|   20 +
 .../hadoop-yarn-services-api/pom.xml|  130 ++
 .../hadoop/yarn/service/webapp/ApiServer.java   |  298 +++
 .../yarn/service/webapp/ApiServerWebApp.java|  161 ++
 .../definition/YARN-Services-Examples.md|  245 +++
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  471 +
 .../src/main/resources/log4j-server.properties  |   76 +
 .../src/main/resources/webapps/api-server/app   |   16 +
 .../src/main/webapp/WEB-INF/web.xml |   36 +
 .../hadoop/yarn/service/ServiceClientTest.java  |  107 ++
 .../hadoop/yarn/service/TestApiServer.java  |  366 
 .../dev-support/findbugs-exclude.xml|   48 +
 .../conf/yarnservice-log4j.properties   |   62 +
 .../examples/httpd-no-dns/httpd-no-dns.json |   62 +
 .../httpd-no-dns/httpd-proxy-no-dns.conf|   24 +
 .../examples/httpd/httpd-proxy.conf |   24 +
 .../examples/httpd/httpd.json   |   55 +
 .../examples/sleeper/sleeper.json   |   15 +
 .../hadoop-yarn-services-core/pom.xml   |  255 +++
 .../hadoop/yarn/service/ClientAMProtocol.java   |   40 +
 .../hadoop/yarn/service/ClientAMService.java|  132 ++
 .../yarn/service/ContainerFailureTracker.java   |   89 +
 .../hadoop/yarn/service/ServiceContext.java |   41 +
 .../hadoop/yarn/service/ServiceMaster.java  |  169 ++
 .../hadoop/yarn/service/ServiceMetrics.java |   94 +
 .../hadoop/yarn/service/ServiceScheduler.java   |  691 +++
 .../yarn/service/api/ServiceApiConstants.java   |   74 +
 .../yarn/service/api/records/Artifact.java  |  168 ++
 .../yarn/service/api/records/BaseResource.java  |   52 +
 .../yarn/service/api/records/Component.java |  430 +
 .../service/api/records/ComponentState.java |   30 +
 .../yarn/service/api/records/ConfigFile.java|  233 +++
 .../yarn/service/api/records/ConfigFormat.java  |   67 +
 .../yarn/service/api/records/Configuration.java |  225 +++
 .../yarn/service/api/records/Container.java |  298 +++
 .../service/api/records/ContainerState.java |   30 +
 .../hadoop/yarn/service/api/records/Error.java  |  129 ++
 .../service/api/records/PlacementPolicy.java|  102 +
 .../service/api/records/ReadinessCheck.java |  183 ++
 .../yarn/service/api/records/Resource.java  |  161 ++
 .../yarn/service/api/records/Service.java   |  390 
 .../yarn/service/api/records/ServiceState.java  |   33 +
 .../yarn/service/api/records/ServiceStatus.java |  148 ++
 .../yarn/service/client/ClientAMProxy.java  |   57 +
 .../yarn/service/client/ServiceClient.java  |  960 ++
 .../yarn/service/component/Component.java   |  584 ++
 .../yarn/service/component/ComponentEvent.java  |   83 +
 .../service/component/ComponentEventType.java   |   27 +
 .../yarn/service/component/ComponentState.java  |   25 +
 .../component/instance/ComponentInstance.java   |  549 ++
 .../instance/ComponentInstanceEvent.java|   58 +
 .../instance/ComponentInstanceEventType.java|   26 +
 .../component/instance/ComponentInstanceId.java |   91 +
 .../instance/ComponentInstanceState.java|   26 +
 .../yarn/service/conf/RestApiConstants.java |   39 +
 .../yarn/service/conf/SliderExitCodes.java  |   88 +
 .../yarn/service/conf/YarnServiceConf.java  |  113 ++
 .../yarn/service/conf/YarnServiceConstants.java |   92 +
 .../containerlaunch/AbstractLauncher.java   |  271 +++
 .../containerlaunch/ClasspathConstructor.java   |  172 ++
 .../containerlaunch/CommandLineBuilder.java |   86 +
 .../containerlaunch/ContainerLaunchService.java |  101 +
 .../containerlaunch/CredentialUtils.java|  319 
 .../containerlaunch/JavaCommandLineBuilder.java |  181 ++
 .../exceptions/BadClusterStateException.java|   

[48/60] [abbrv] hadoop git commit: Rebase to trunk

[jianhe]

Rebase to trunk


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3c30b1a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3c30b1a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3c30b1a9

Branch: refs/heads/trunk
Commit: 3c30b1a97d4d93a5714a91db90749c2a581bfdc4
Parents: 9273480
Author: Jian He 
Authored: Fri Oct 13 11:15:02 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3c30b1a9/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 727384c..cc269d8 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -51,7 +51,6 @@ function hadoop_usage
   hadoop_add_subcommand "router" daemon "run the Router daemon"
   hadoop_add_subcommand "schedulerconf" client "Updates scheduler 
configuration"
   hadoop_add_subcommand "scmadmin" admin "SharedCacheManager admin tools"
-  hadoop_add_subcommand "sharedcachemanager" daemon "run the 
SharedCacheManager daemon"
   hadoop_add_subcommand "service" client "deploy/manage a service on YARN"
   hadoop_add_subcommand "sharedcachemanager" daemon "run the 
SharedCacheManager daemon"
   hadoop_add_subcommand "timelinereader" client "run the timeline reader 
server"


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[05/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
deleted file mode 100644
index b3fdef9..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * 
- *   Usage: create an instance, then when creating a Jersey Client
- *   pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
- */
-public class UgiJerseyBinding implements
-HttpURLConnectionFactory {
-  private static final Logger log =
-  LoggerFactory.getLogger(UgiJerseyBinding.class);
-
-  private final UrlConnectionOperations operations;
-  private final URLConnectionClientHandler handler;
-
-  /**
-   * Construct an instance
-   * @param operations operations instance
-   */
-  @SuppressWarnings("ThisEscapedInObjectConstruction")
-  public UgiJerseyBinding(UrlConnectionOperations operations) {
-Preconditions.checkArgument(operations != null, "Null operations");
-this.operations = operations;
-handler = new URLConnectionClientHandler(this);
-  }
-
-  /**
-   * Create an instance off the configuration. The SPNEGO policy
-   * is derived from the current UGI settings.
-   * @param conf config
-   */
-  public UgiJerseyBinding(Configuration conf) {
-this(new UrlConnectionOperations(conf));
-  }
-
-  /**
-   * Get a URL connection. 
-   * @param url URL to connect to
-   * @return the connection
-   * @throws IOException any problem. {@link AuthenticationException} 
-   * errors are wrapped
-   */
-  @Override
-  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
-try {
-  // open a connection handling status codes and so redirections
-  // but as it opens a connection, it's less useful than you think.
-
-  return operations.openConnection(url);
-} catch (AuthenticationException e) {
-  throw new IOException(e);
-}
-  }
-
-  public UrlConnectionOperations getOperations() {
-return operations;
-  }
-
-  public URLConnectionClientHandler getHandler() {
-return handler;
-  }
-  
-  /**
-   * Get the SPNEGO flag (as found in the operations instance
-   * @return the spnego policy
-   */
-  public boolean isUseSpnego() {
-return operations.isUseSpnego();
-  }
-
-
-  /**
-   * Uprate error codes 400 and up into faults; 
-   * 
-   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
-   */
-  public static IOException uprateFaults(HttpVerb verb, String url,
- 

[09/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index c2866cf..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- * 
- * --instance {app name}, if  a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles 
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
-commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
-  public static final String USAGE =
-  "Usage: " + SliderActions.ACTION_REGISTRY
-  + " ("
-  + Arguments.ARG_LIST + "|"
-  + Arguments.ARG_LISTCONF + "|"
-  + Arguments.ARG_LISTEXP + "|"
-  + Arguments.ARG_LISTFILES + "|"
-  + Arguments.ARG_GETCONF + "|"
-  + Arguments.ARG_GETEXP + "> "
-  + Arguments.ARG_NAME + "  "
-  + " )"
-  + "[" + Arguments.ARG_VERBOSE + "] "
-  + "[" + Arguments.ARG_USER + "] "
-  + "[" + Arguments.ARG_OUTPUT + "  ] "
-  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
-  + "[" + Arguments.ARG_FORMAT + " ] "
-  + System.getProperty("line.separator")
-  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
-  ;
-  public ActionRegistryArgs() {
-  }
-
-  public ActionRegistryArgs(String name) {
-this.name = name;
-  }
-
-  @Override
-  public String getActionName() {
-return ACTION_REGISTRY;
-  }
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  @Override
-  public int getMinParams() {
-return 0;
-  }
-  
-  @Parameter(names = {ARG_LIST}, 
-  description = "list services")
-  public boolean list;
-
-  @Parameter(names = {ARG_LISTCONF}, 
-  description = "list configurations")
-  public boolean listConf;
-
-  @Parameter(names = {ARG_GETCONF},
-  description = "get configuration")
-  public String getConf;
-
-  @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
-  public boolean listExports;
-
-  @Parameter(names = {ARG_GETEXP},
- description = "get export")
-  public String getExport;
-
-  @Parameter(names = {ARG_LISTFILES},
-  description = "list files")
-  public String listFiles;
-
-  @Parameter(names = {ARG_GETFILES},
-  description = "get files")
-  public String getFiles;
-
-  //--format 
-  @Parameter(names = ARG_FORMAT,
-  description = "Format for a response: ")
-  public String format = ConfigFormat.XML.toString() ;
-
-  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
-  description = "Output destination")
-  public File out;
-
-  @Parameter(names 

[06/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
deleted file mode 100644
index 83c9961..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import 
org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class ProviderFactory {
-  protected static final Logger LOG =
-  LoggerFactory.getLogger(ProviderFactory.class);
-
-  protected ProviderFactory() {}
-
-  public abstract AbstractClientProvider createClientProvider();
-
-  public abstract ProviderService createServerProvider();
-
-  public static synchronized ProviderService getProviderService(Artifact
-  artifact) {
-return createSliderProviderFactory(artifact).createServerProvider();
-  }
-
-  public static synchronized AbstractClientProvider getClientProvider(Artifact
-  artifact) {
-return createSliderProviderFactory(artifact).createClientProvider();
-  }
-
-  /**
-   * Create a provider for a specific application
-   * @param artifact artifact
-   * @return provider factory
-   */
-  public static synchronized ProviderFactory createSliderProviderFactory(
-  Artifact artifact) {
-if (artifact == null || artifact.getType() == null) {
-  LOG.debug("Loading service provider type default");
-  return DefaultProviderFactory.getInstance();
-}
-LOG.debug("Loading service provider type {}", artifact.getType());
-switch (artifact.getType()) {
-  // TODO add handling for custom types?
-  // TODO handle application
-  case DOCKER:
-return DockerProviderFactory.getInstance();
-  case TARBALL:
-return TarballProviderFactory.getInstance();
-  default:
-throw new IllegalArgumentException(String.format("Resolution error, " +
-"%s should not be passed to createSliderProviderFactory",
-artifact.getType()));
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
deleted file mode 100644
index 9ef0176..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  

[10/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
deleted file mode 100644
index 1049698..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;

[14/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
new file mode 100644
index 000..be36335
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for ServiceApiUtil helper methods.
+ */
+public class TestServiceApiUtil {
+  private static final Logger LOG = LoggerFactory
+  .getLogger(TestServiceApiUtil.class);
+  private static final String EXCEPTION_PREFIX = "Should have thrown " +
+  "exception: ";
+  private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+  "exception: ";
+
+  private static final String LEN_64_STR =
+  "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
+
+  private static final YarnConfiguration CONF_DEFAULT_DNS = new
+  YarnConfiguration();
+  private static final YarnConfiguration CONF_DNS_ENABLED = new
+  YarnConfiguration();
+
+  @BeforeClass
+  public static void init() {
+CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
+  }
+
+  @Test(timeout = 9)
+  public void testResourceValidation() throws Exception {
+assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
+.length());
+
+SliderFileSystem sfs = initMock(null);
+
+Service app = new Service();
+
+// no name
+try {
+  ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+  Assert.fail(EXCEPTION_PREFIX + "service with no name");
+} catch (IllegalArgumentException e) {
+  assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+}
+
+// bad format name
+String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
+for (String badName : badNames) {
+  app.setName(badName);
+  try {
+ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName);
+  } catch (IllegalArgumentException e) {
+

[21/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
new file mode 100644
index 000..3e53418
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+
+import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
+import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+import java.io.File;
+
+/**
+ * Registry actions
+ * 
+ * --instance {app name}, if  a / is in it, refers underneath?
+ * --dest {destfile}
+ * --list : list instances of slider service
+ * --listfiles 
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+commandDescription = DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+
+  public static final String USAGE =
+  "Usage: " + SliderActions.ACTION_REGISTRY
+  + " ("
+  + Arguments.ARG_LIST + "|"
+  + Arguments.ARG_LISTCONF + "|"
+  + Arguments.ARG_LISTEXP + "|"
+  + Arguments.ARG_LISTFILES + "|"
+  + Arguments.ARG_GETCONF + "|"
+  + Arguments.ARG_GETEXP + "> "
+  + Arguments.ARG_NAME + "  "
+  + " )"
+  + "[" + Arguments.ARG_VERBOSE + "] "
+  + "[" + Arguments.ARG_USER + "] "
+  + "[" + Arguments.ARG_OUTPUT + "  ] "
+  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
+  + "[" + Arguments.ARG_FORMAT + " ] "
+  + System.getProperty("line.separator")
+  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+  ;
+  public ActionRegistryArgs() {
+  }
+
+  public ActionRegistryArgs(String name) {
+this.name = name;
+  }
+
+  @Override
+  public String getActionName() {
+return ACTION_REGISTRY;
+  }
+
+  /**
+   * Get the min #of params expected
+   * @return the min number of params in the {@link #parameters} field
+   */
+  @Override
+  public int getMinParams() {
+return 0;
+  }
+  
+  @Parameter(names = {ARG_LIST}, 
+  description = "list services")
+  public boolean list;
+
+  @Parameter(names = {ARG_LISTCONF}, 
+  description = "list configurations")
+  public boolean listConf;
+
+  @Parameter(names = {ARG_GETCONF},
+  description = "get configuration")
+  public String getConf;
+
+  @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+  public boolean listExports;
+
+  @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+  public String getExport;
+
+  @Parameter(names = {ARG_LISTFILES},
+  description = "list files")
+  public String listFiles;
+
+  @Parameter(names = {ARG_GETFILES},
+  description = "get files")
+  public String getFiles;
+
+  //--format 
+  @Parameter(names = ARG_FORMAT,
+  description = "Format for a response: ")
+  public String format = ConfigFormat.XML.toString() ;
+
+  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
+  description = "Output destination")
+  public File out;
+
+  

[01/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk dcd99c4b9 -> cbc632d9a


http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
deleted file mode 100644
index 8310530..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestLoadExampleAppJson extends Assert {
-  private String resource;
-
-  public TestLoadExampleAppJson(String resource) {
-this.resource = resource;
-  }
-
-  @Parameterized.Parameters
-  public static Collection filenames() {
-String[][] stringArray = new String[ExampleAppJson
-.ALL_EXAMPLE_RESOURCES.size()][1];
-int i = 0;
-for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
-  stringArray[i++][0] = s;
-}
-return Arrays.asList(stringArray);
-  }
-
-  @Test
-  public void testLoadResource() throws Throwable {
-try {
-  Application application = JSON_SER_DESER.fromResource(resource);
-
-  SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-  FileSystem mockFs = createNiceMock(FileSystem.class);
-  expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-  expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-  new Path("cluster_dir_path")).anyTimes();
-  replay(sfs, mockFs);
-
-  ServiceApiUtil.validateAndResolveApplication(application, sfs,
-  new YarnConfiguration());
-} catch (Exception e) {
-  throw new Exception("exception loading " + resource + ":" + 
e.toString());
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
deleted file mode 100644
index 98c78d3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional 

[30/60] [abbrv] hadoop git commit: YARN-7126. Create introductory site documentation for YARN native services. Contributed by Gour Saha

[jianhe]

YARN-7126. Create introductory site documentation for YARN native services. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e660e52a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e660e52a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e660e52a

Branch: refs/heads/trunk
Commit: e660e52a048eae39f6b57f3c085ebbd34c04dd17
Parents: 7dd4490
Author: Jian He 
Authored: Fri Sep 1 16:19:31 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:14 2017 -0800

--
 LICENSE.txt |  1 +
 .../native-services/NativeServicesIntro.md  | 96 +++-
 2 files changed, 96 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e660e52a/LICENSE.txt
--
diff --git a/LICENSE.txt b/LICENSE.txt
index 67472b6..b0cef03 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1777,6 +1777,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH 
DAMAGE.
 The binary distribution of this product bundles these dependencies under the
 following license:
 FindBugs-jsr305 3.0.0
+dnsjava 2.1.7, Copyright (c) 1998-2011, Brian Wellington. All rights reserved.
 

 (2-clause BSD)
 Redistribution and use in source and binary forms, with or without

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e660e52a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
index 89fefe9..e6a4e91 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesIntro.md
@@ -10,4 +10,98 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. See accompanying LICENSE file.
--->
\ No newline at end of file
+-->
+
+# Introduction: YARN Native Services
+
+## Overview
+YARN Native Services provides first class framework support and APIs to host 
long running services natively in YARN. In addition to launching services, the 
new APIs support performing lifecycle management operations, such as flex 
service components up/down, manage lifetime, upgrade the service to a newer 
version, and stop/restart/delete the service.
+
+The native services capabilities are built on the existing low-level resource 
management API provided by YARN that can support any type of application. Other 
application frameworks like Hadoop MapReduce already expose higher level APIs 
that users can leverage to run applications on top of YARN. With the advent of 
containerization technologies like Docker, providing first class support and 
APIs for long running services at the framework level made sense.
+
+Relying on a framework has the advantage of exposing a simpler usage model to 
the user by enabling service configuration and launch through specification 
(without writing new code), as well as hiding complex low-level details 
including state management and fault-tolerance etc. Users/operators of existing 
services typically like to avoid modifying an existing service to be aware of 
YARN. With first class support capable of running a single Docker image as well 
as complex assemblies comprised of multiple Docker images, there is no need for 
service owners to be aware of YARN. Developers of new services do not have to 
worry about YARN internals and only need to focus on containerization of their 
service(s).
+
+## First class support for services
+In order to natively provide first class support for long running services, 
several new features and improvements have been made at the framework level.
+
+### Incorporate Apache Slider into Apache YARN
+Apache Slider, which existed as a separate incubator project has been merged 
into YARN to kick start the first class support. Apache Slider is a universal 
Application Master (AM) which had several key features built in - fault 
tolerance of service containers and AM, work-preserving AM restarts, service 
logs management, service management like flex up/down, stop/start, and rolling 
upgrade to newer service versions, etc. Of course lot more work has been done 
on top of what Apache Slider brought in, details 

[17/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 000..add2475
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+  private static final Log LOG =
+  LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+  private final RegistryOperations registryOperations;
+  private final String user;
+  private final String sliderServiceClass;
+  private final String instanceName;
+  /**
+   * Record used where the service registered itself.
+   * Null until the service is registered
+   */
+  private ServiceRecord selfRegistration;
+
+  /**
+   * Path where record was registered
+   * Null until the service is registered
+   */
+  private String selfRegistrationPath;
+
+  public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+  String user,
+  String sliderServiceClass,
+  String instanceName,
+  ApplicationAttemptId applicationAttemptId) {
+Preconditions.checkArgument(registryOperations != null,
+"null registry operations");
+Preconditions.checkArgument(user != null, "null user");
+Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+"unset service class");
+Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+"instanceName");
+Preconditions.checkArgument(applicationAttemptId != null,
+"null applicationAttemptId");
+this.registryOperations = registryOperations;
+this.user = user;
+this.sliderServiceClass = sliderServiceClass;
+this.instanceName = instanceName;
+  }
+
+  public String getUser() {
+return user;
+  }
+
+
+  private void setSelfRegistration(ServiceRecord selfRegistration) {
+this.selfRegistration = selfRegistration;
+  }
+
+  /**
+   * Get the path to where the service has registered itself.
+   * Null until the service is registered
+   * @return the service registration path.
+   */
+  public String getSelfRegistrationPath() {
+return selfRegistrationPath;
+  }
+
+  /**
+   * Get the absolute path to where the service has registered itself.
+   * This includes the base registry path
+   * Null until the service is registered
+   * @return the service 

[15/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 000..6e6f4dd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+  private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+  /**

[18/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 000..684f655
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+  protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+  .class);
+
+  private MonitorUtils() {
+  }
+
+  public static String toPlural(int val) {
+return val != 1 ? "s" : "";
+  }
+
+  /**
+   * Convert milliseconds to human time -the exact format is unspecified
+   * @param milliseconds a time in milliseconds
+   * @return a time that is converted to human intervals
+   */
+  public static String millisToHumanTime(long milliseconds) {
+StringBuilder sb = new StringBuilder();
+// Send all output to the Appendable object sb
+Formatter formatter = new Formatter(sb, Locale.US);
+
+long s = Math.abs(milliseconds / 1000);
+long m = Math.abs(milliseconds % 1000);
+if (milliseconds > 0) {
+  formatter.format("%d.%03ds", s, m);
+} else if (milliseconds == 0) {
+  formatter.format("0");
+} else {
+  formatter.format("-%d.%03ds", s, m);
+}
+return sb.toString();
+  }
+
+  public static Probe getProbe(ReadinessCheck readinessCheck) {
+if (readinessCheck == null) {
+  return null;
+}
+if (readinessCheck.getType() == null) {
+  return null;
+}
+try {
+  switch (readinessCheck.getType()) {
+  case HTTP:
+return HttpProbe.create(readinessCheck.getProps());
+  case PORT:
+return PortProbe.create(readinessCheck.getProps());
+  default:
+return null;
+  }
+} catch (Throwable t) {
+  throw new IllegalArgumentException("Error creating readiness check " +
+  t);
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 000..aba5859
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You 

[28/60] [abbrv] hadoop git commit: Rebase onto latest trunk. minor conflicts

[jianhe]

Rebase onto latest trunk. minor conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd96c4c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd96c4c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd96c4c2

Branch: refs/heads/trunk
Commit: bd96c4c235c15c1df6ba37544f8e15004ad1d09d
Parents: 59fe98c
Author: Jian He 
Authored: Wed Aug 30 22:48:35 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:13 2017 -0800

--
 .../java/org/apache/hadoop/yarn/util/Apps.java  |  2 ++
 .../rmapp/attempt/RMAppAttemptImpl.java | 35 ++--
 2 files changed, 4 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 5efbd2e..685c6d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -150,6 +150,8 @@ public class Apps {
   }
 
   // Check if should black list the node based on container exit status
+  @Private
+  @Unstable
   public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
 switch (exitStatus) {
 case ContainerExitStatus.PREEMPTED:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4997bc6..d3fbcb5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BoundedAppender;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
@@ -1539,38 +1540,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, 
Recoverable {
 }
   }
 
-  private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
-switch (exitStatus) {
-case ContainerExitStatus.PREEMPTED:
-case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
-case ContainerExitStatus.KILLED_BY_APPMASTER:
-case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
-case ContainerExitStatus.ABORTED:
-  // Neither the app's fault nor the system's fault. This happens by 
design,
-  // so no need for skipping nodes
-  return false;
-case ContainerExitStatus.DISKS_FAILED:
-  // This container is marked with this exit-status means that the node is
-  // already marked as unhealthy given that most of the disks failed. So, 
no
-  // need for any explicit skipping of nodes.
-  return false;
-case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
-case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
-  // No point in skipping the node as it's not the system's fault
-  return false;
-case ContainerExitStatus.SUCCESS:
-  return false;
-case ContainerExitStatus.INVALID:
-  // Ideally, this shouldn't be considered for skipping a node. But in
-  // reality, it seems like there are cases where we are not setting
-  // exit-code correctly and so it's better to be conservative. See
-  // YARN-4284.
-  return true;
-default:
-  return true;
-}
-  }
-
   private static final class UnmanagedAMAttemptSavedTransition
 extends 

[04/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
deleted file mode 100644
index fe8cce8..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; 
this allows
- * the same method
- */
-public class ConfigHelper {
-  private static final Logger log = 
LoggerFactory.getLogger(ConfigHelper.class);
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Map map,
-  String origin) throws BadConfigException {
-addConfigMap(config, map.entrySet(), origin);
-  }
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Iterable> map,
-  String origin) throws BadConfigException {
-for (Map.Entry mapEntry : map) {
-  String key = mapEntry.getKey();
-  String value = mapEntry.getValue();
-  if (value == null) {
-throw new BadConfigException("Null value for property " + key);
-  }
-  config.set(key, value, origin);
-}
-  }
-
-  /**
-   * Convert to an XML string
-   * @param conf configuration
-   * @return conf
-   * @throws IOException
-   */
-  public static String toXml(Configuration conf) throws IOException {
-StringWriter writer = new StringWriter();
-conf.writeXml(writer);
-return writer.toString();
-  }
-
-
-  /**
-   * Register a resource as a default resource.
-   * Do not attempt to use this unless you understand that the
-   * order in which default resources are loaded affects the outcome,
-   * and that subclasses of Configuration often register new default
-   * resources
-   * @param resource the resource name
-   * @return the URL or null
-   */
-  public static URL registerDefaultResource(String resource) {
-URL resURL = getResourceUrl(resource);
-if (resURL != null) {
-  Configuration.addDefaultResource(resource);
-}
-return resURL;
-  }
-
-  /**
-   * Load a configuration from a resource on this classpath.
-   * If the resource is not found, an empty configuration is returned
-   * @param resource the resource name
-   * @return the loaded configuration.
-   */
-  public static Configuration loadFromResource(String resource) {
-Configuration conf = new Configuration(false);
-URL resURL = getResourceUrl(resource);
-if (resURL != null) {
-  log.debug("loaded resources from {}", resURL);
-  conf.addResource(resource);
-} else{
-  log.debug("failed to find {} on 

[24/60] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
new file mode 100644
index 000..fb2fd16
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -0,0 +1,654 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import 

[27/58] [abbrv] hadoop git commit: Rebase onto latest trunk. minor conflicts

[jianhe]

Rebase onto latest trunk. minor conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd96c4c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd96c4c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd96c4c2

Branch: refs/heads/yarn-native-services
Commit: bd96c4c235c15c1df6ba37544f8e15004ad1d09d
Parents: 59fe98c
Author: Jian He 
Authored: Wed Aug 30 22:48:35 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:13 2017 -0800

--
 .../java/org/apache/hadoop/yarn/util/Apps.java  |  2 ++
 .../rmapp/attempt/RMAppAttemptImpl.java | 35 ++--
 2 files changed, 4 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 5efbd2e..685c6d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -150,6 +150,8 @@ public class Apps {
   }
 
   // Check if should black list the node based on container exit status
+  @Private
+  @Unstable
   public static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
 switch (exitStatus) {
 case ContainerExitStatus.PREEMPTED:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd96c4c2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4997bc6..d3fbcb5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -107,6 +107,7 @@ import org.apache.hadoop.yarn.state.MultipleArcTransition;
 import org.apache.hadoop.yarn.state.SingleArcTransition;
 import org.apache.hadoop.yarn.state.StateMachine;
 import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
 import org.apache.hadoop.yarn.util.BoundedAppender;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 
@@ -1539,38 +1540,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, 
Recoverable {
 }
   }
 
-  private static boolean shouldCountTowardsNodeBlacklisting(int exitStatus) {
-switch (exitStatus) {
-case ContainerExitStatus.PREEMPTED:
-case ContainerExitStatus.KILLED_BY_RESOURCEMANAGER:
-case ContainerExitStatus.KILLED_BY_APPMASTER:
-case ContainerExitStatus.KILLED_AFTER_APP_COMPLETION:
-case ContainerExitStatus.ABORTED:
-  // Neither the app's fault nor the system's fault. This happens by 
design,
-  // so no need for skipping nodes
-  return false;
-case ContainerExitStatus.DISKS_FAILED:
-  // This container is marked with this exit-status means that the node is
-  // already marked as unhealthy given that most of the disks failed. So, 
no
-  // need for any explicit skipping of nodes.
-  return false;
-case ContainerExitStatus.KILLED_EXCEEDED_VMEM:
-case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
-  // No point in skipping the node as it's not the system's fault
-  return false;
-case ContainerExitStatus.SUCCESS:
-  return false;
-case ContainerExitStatus.INVALID:
-  // Ideally, this shouldn't be considered for skipping a node. But in
-  // reality, it seems like there are cases where we are not setting
-  // exit-code correctly and so it's better to be conservative. See
-  // YARN-4284.
-  return true;
-default:
-  return true;
-}
-  }
-
   private static final class UnmanagedAMAttemptSavedTransition
 

[23/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 000..c64b1b5
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+  private Integer code = null;
+  private String message = null;
+  private String fields = null;
+
+  /**
+   **/
+  public Error code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  /**
+   **/
+  public Error message(String message) {
+this.message = message;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("message")
+  public String getMessage() {
+return message;
+  }
+
+  public void setMessage(String message) {
+this.message = message;
+  }
+
+  /**
+   **/
+  public Error fields(String fields) {
+this.fields = fields;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("fields")
+  public String getFields() {
+return fields;
+  }
+
+  public void setFields(String fields) {
+this.fields = fields;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+Error error = (Error) o;
+return Objects.equals(this.code, error.code)
+&& Objects.equals(this.message, error.message)
+&& Objects.equals(this.fields, error.fields);
+  }
+
+  @Override
+  public int hashCode() {
+return Objects.hash(code, message, fields);
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder();
+sb.append("class Error {\n");
+
+sb.append("code: ").append(toIndentedString(code)).append("\n");
+sb.append("message: ").append(toIndentedString(message)).append("\n");
+sb.append("fields: ").append(toIndentedString(fields)).append("\n");
+sb.append("}");
+return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+if (o == null) {
+  return "null";
+}
+return o.toString().replace("\n", "\n");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
--
diff --git 

[46/58] [abbrv] hadoop git commit: YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi

[jianhe]

YARN-7198. Add jsvc support for RegistryDNS. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92734800
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92734800
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92734800

Branch: refs/heads/yarn-native-services
Commit: 92734800844b93bb19f3c0fca6be88d2801f7b1b
Parents: b57144a
Author: Jian He 
Authored: Fri Oct 13 10:49:38 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn|  6 ++
 .../hadoop-yarn/conf/yarn-env.sh| 12 +++
 .../hadoop-yarn/hadoop-yarn-registry/pom.xml| 73 +-
 .../dns/PrivilegedRegistryDNSStarter.java   | 80 
 .../hadoop/registry/server/dns/RegistryDNS.java | 45 +++
 .../registry/server/dns/RegistryDNSServer.java  | 57 +-
 .../src/site/markdown/YarnCommands.md   |  8 ++
 .../src/site/markdown/registry/index.md |  1 +
 .../markdown/registry/registry-configuration.md |  4 +-
 .../site/markdown/yarn-service/RegistryDNS.md   | 40 +-
 .../markdown/yarn-service/ServiceDiscovery.md   | 21 -
 11 files changed, 282 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 05d892b..727384c 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -45,6 +45,7 @@ function hadoop_usage
   hadoop_add_subcommand "nodemanager" daemon "run a nodemanager on each worker"
   hadoop_add_subcommand "proxyserver" daemon "run the web app proxy server"
   hadoop_add_subcommand "queue" client "prints queue information"
+  hadoop_add_subcommand "registrydns" daemon "run the registry DNS server"
   hadoop_add_subcommand "resourcemanager" daemon "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" admin "admin tools"
   hadoop_add_subcommand "router" daemon "run the Router daemon"
@@ -143,6 +144,11 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
 queue)
   HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
 ;;
+registrydns)
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+  
HADOOP_SECURE_CLASSNAME='org.apache.hadoop.registry.server.dns.PrivilegedRegistryDNSStarter'
+  
HADOOP_CLASSNAME='org.apache.hadoop.registry.server.dns.RegistryDNSServer'
+;;
 resourcemanager)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'

http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh 
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index 90a87bf..4bd1d3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -160,3 +160,15 @@
 # See ResourceManager for some examples
 #
 #export YARN_APISERVER_OPTS="-verbose:gc -XX:+PrintGCDetails 
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps 
-Xloggc:${HADOOP_LOG_DIR}/gc-apiserver.log-$(date +'%Y%m%d%H%M')"
+
+###
+# Registry DNS specific parameters
+###
+# For privileged registry DNS, user to run as after dropping privileges
+# This will replace the hadoop.id.str Java property in secure mode.
+# export YARN_REGISTRYDNS_SECURE_USER=yarn
+
+# Supplemental options for privileged registry DNS
+# By default, Hadoop uses jsvc which needs to know to launch a
+# server jvm.
+# export YARN_REGISTRYDNS_SECURE_EXTRA_OPTS="-jvm server"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/92734800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index e083312..4e805cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -34,6 +34,21 @@
   
 
 
+  org.slf4j
+  slf4j-api
+
+
+
+  org.apache.hadoop
+  hadoop-auth
+
+
+
+  org.apache.hadoop
+  hadoop-annotations
+
+
+
   org.apache.hadoop
   hadoop-yarn-api
 
@@ -70,14 +85,68 @@
 
 
 
+  org.apache.zookeeper
+  zookeeper
+
+
+
+  org.apache.curator
+  

[41/58] [abbrv] hadoop git commit: YARN-6626. Embed REST API service into RM. Contributed by Eric Yang

[jianhe]

YARN-6626. Embed REST API service into RM. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e677fa0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e677fa0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e677fa0

Branch: refs/heads/yarn-native-services
Commit: 9e677fa05c6a822cf034bffd25bbee0bad84084b
Parents: 8851209
Author: Jian He 
Authored: Thu Sep 28 16:29:22 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:17 2017 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  2 ++
 .../hadoop/yarn/service/webapp/ApiServer.java   |  7 
 .../hadoop-yarn-services-core/pom.xml   |  6 
 .../yarn/service/api/records/Artifact.java  |  8 +
 .../yarn/service/api/records/ConfigFile.java|  8 +
 .../service/api/records/ReadinessCheck.java |  8 +
 .../src/main/resources/yarn-default.xml |  8 +
 .../server/resourcemanager/webapp/RMWebApp.java | 19 +++
 .../site/markdown/yarn-service/QuickStart.md| 34 +---
 9 files changed, 89 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 640e86e..4799137 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -336,6 +336,8 @@ public class YarnConfiguration extends Configuration {
 
   public static final String YARN_WEBAPP_UI2_WARFILE_PATH = "yarn."
   + "webapp.ui2.war-file-path";
+  public static final String YARN_API_SERVICES_ENABLE = "yarn."
+  + "webapp.api-service.enable";
 
   public static final String RM_RESOURCE_TRACKER_ADDRESS =
 RM_PREFIX + "resource-tracker.address";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e8286ef..89b020d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -17,6 +17,7 @@
 
 package org.apache.hadoop.yarn.service.webapp;
 
+import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
@@ -57,6 +58,12 @@ import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.*;
 @Singleton
 @Path(CONTEXT_ROOT)
 public class ApiServer {
+
+  @Inject
+  public ApiServer(Configuration conf) {
+super();
+  }
+
   private static final Logger LOG =
   LoggerFactory.getLogger(ApiServer.class);
   private static Configuration YARN_CONFIG = new YarnConfiguration();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e677fa0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
index fb07edc..205a64d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/pom.xml
@@ -245,12 +245,6 @@
 
 
 
-  org.apache.hadoop
-  hadoop-yarn-server-resourcemanager
-  test
-
-
-
   org.apache.curator
   curator-test
   test

[43/58] [abbrv] hadoop git commit: YARN-7202. Add UT for api-server. Contributed by Eric Yang

[jianhe]

YARN-7202. Add UT for api-server. Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b57144a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b57144a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b57144a1

Branch: refs/heads/yarn-native-services
Commit: b57144a1726e0b02abbdc3e8f37769d3f7c039c8
Parents: b8a7ef1
Author: Jian He 
Authored: Thu Oct 12 10:57:35 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 .../hadoop-yarn-services-api/pom.xml|  34 +-
 .../hadoop/yarn/service/webapp/ApiServer.java   |  21 +-
 .../yarn/service/webapp/ApiServerWebApp.java|   2 +-
 .../hadoop/yarn/service/ServiceClientTest.java  | 107 ++
 .../hadoop/yarn/service/TestApiServer.java  | 366 +++
 .../hadoop/yarn/service/ServiceTestUtils.java   | 171 +
 .../yarn/service/TestYarnNativeServices.java| 162 +---
 7 files changed, 690 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index 74d9681..ddea2a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -41,7 +41,6 @@
 
 
 
-
   
 org.apache.maven.plugins
 maven-jar-plugin
@@ -66,17 +65,6 @@
   
 
   
-
-  
-org.apache.maven.plugins
-maven-surefire-plugin
-
-  
-${java.home}
-  
-
-  
-
 
   
 
@@ -84,6 +72,7 @@
   
 
   
+
 
   org.apache.hadoop
   hadoop-yarn-services-core
@@ -116,5 +105,26 @@
   javax.ws.rs
   jsr311-api
 
+
+  org.mockito
+  mockito-all
+  test
+
+
+
+
+
+
+
+  org.apache.hadoop
+  hadoop-common
+  test-jar
+
+
+  junit
+  junit
+  test
+
+
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b57144a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 4fe12e3..5773069 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -165,12 +165,12 @@ public class ApiServer {
   } else {
 LOG.info("Successfully stopped service {}", appName);
   }
-  return Response.status(Status.NO_CONTENT).build();
+  return Response.status(Status.OK).build();
 } catch (ApplicationNotFoundException e) {
   ServiceStatus serviceStatus = new ServiceStatus();
   serviceStatus.setDiagnostics(
-  "Service " + appName + " not found " + e.getMessage());
-  return Response.status(Status.NOT_FOUND).entity(serviceStatus)
+  "Service " + appName + " is not found in YARN: " + e.getMessage());
+  return Response.status(Status.BAD_REQUEST).entity(serviceStatus)
   .build();
 } catch (Exception e) {
   ServiceStatus serviceStatus = new ServiceStatus();
@@ -245,7 +245,8 @@ public class ApiServer {
 // flex a single component app
 if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
 .hasComponent(updateServiceData)) {
-  Component defaultComp = 
ServiceApiUtil.createDefaultComponent(updateServiceData);
+  Component defaultComp = ServiceApiUtil
+  .createDefaultComponent(updateServiceData);
   return updateComponent(updateServiceData.getName(), 
defaultComp.getName(),
   defaultComp);
 }
@@ -291,4 +292,16 @@ public class ApiServer {
   .entity(status).build();
 }
   }
+
+  /**
+   * Used by negative test case.
+   *
+   * @param mockServerClient - A mocked version of 

[44/58] [abbrv] hadoop git commit: YARN-7323. Data structure update in service REST API. Contributed by Jian He

[jianhe]

YARN-7323. Data structure update in service REST API. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68acd88d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68acd88d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68acd88d

Branch: refs/heads/yarn-native-services
Commit: 68acd88dcbfe03a0134c60b5398dfaa31ad2b786
Parents: 3c30b1a
Author: Billie Rinaldi 
Authored: Tue Oct 17 11:50:16 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:18 2017 -0800

--
 .../hadoop/yarn/service/webapp/ApiServer.java   |  9 ---
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 67 -
 .../hadoop/yarn/service/ServiceScheduler.java   |  1 +
 .../yarn/service/api/records/Component.java | 22 +-
 .../service/api/records/ComponentState.java | 30 
 .../yarn/service/api/records/Container.java | 15 ++--
 .../yarn/service/api/records/Service.java   | 76 
 .../yarn/service/api/records/ServiceState.java  |  2 +-
 .../yarn/service/client/ServiceClient.java  | 45 
 .../yarn/service/component/Component.java   | 12 
 .../component/instance/ComponentInstance.java   | 48 +
 .../yarn/service/conf/RestApiConstants.java |  4 --
 .../yarn/service/utils/ServiceApiUtil.java  | 53 ++
 .../hadoop/yarn/service/ServiceTestUtils.java   |  4 +-
 .../hadoop/yarn/service/TestServiceApiUtil.java | 74 ++-
 .../yarn/service/TestYarnNativeServices.java|  4 +-
 .../service/conf/examples/app-override.json |  8 ++-
 .../hadoop/yarn/service/conf/examples/app.json  |  8 ++-
 .../yarn/service/conf/examples/external0.json   | 15 ++--
 .../markdown/yarn-service/YarnServiceAPI.md | 22 +++---
 20 files changed, 232 insertions(+), 287 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 5773069..1bb6c93 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -242,15 +242,6 @@ public class ApiServer {
   return updateLifetime(appName, updateServiceData);
 }
 
-// flex a single component app
-if (updateServiceData.getNumberOfContainers() != null && !ServiceApiUtil
-.hasComponent(updateServiceData)) {
-  Component defaultComp = ServiceApiUtil
-  .createDefaultComponent(updateServiceData);
-  return updateComponent(updateServiceData.getName(), 
defaultComp.getName(),
-  defaultComp);
-}
-
 // If nothing happens consider it a no-op
 return Response.status(Status.NO_CONTENT).build();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/68acd88d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index b9b5b3a..cc76259 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -19,28 +19,17 @@ swagger: '2.0'
 info:
   title: "YARN Simplified API layer for services"
   description: |
-Bringing a new service on YARN today is not a simple experience. The APIs 
of
-existing frameworks are either too low level (native YARN), require writing
-new code (for frameworks with programmatic APIs) or writing a complex spec
-(for declarative frameworks). In addition to building 

[56/58] [abbrv] hadoop git commit: YARN-6394. Support specifying YARN related params in the service REST API. Contributed by Jian He

[jianhe]

YARN-6394. Support specifying YARN related params in the service REST API. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6c4bd74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6c4bd74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6c4bd74

Branch: refs/heads/yarn-native-services
Commit: a6c4bd74b636d0715ffb4c1aca9ef4e49201aa02
Parents: a288b0a
Author: Billie Rinaldi 
Authored: Fri Nov 3 11:25:57 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:20 2017 -0800

--
 .../definition/YARN-Services-Examples.md|   8 +-
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |   6 +-
 .../examples/httpd-no-dns/httpd-no-dns.json |   6 +-
 .../examples/httpd/httpd.json   |   4 +-
 .../hadoop/yarn/service/ServiceMaster.java  |   6 +
 .../hadoop/yarn/service/ServiceMetrics.java |   4 -
 .../hadoop/yarn/service/ServiceScheduler.java   |  15 --
 .../yarn/service/api/ServiceApiConstants.java   |   4 +-
 .../yarn/service/api/records/ConfigFile.java|  32 ++--
 .../yarn/service/api/records/Configuration.java |   4 +-
 .../service/api/records/ReadinessCheck.java |  24 +--
 .../yarn/service/client/ServiceClient.java  |  61 ---
 .../yarn/service/component/Component.java   |  29 ++--
 .../component/instance/ComponentInstance.java   |  37 +++--
 .../yarn/service/conf/YarnServiceConf.java  |  14 +-
 .../yarn/service/conf/YarnServiceConstants.java |   2 +
 .../service/monitor/probe/MonitorUtils.java |   4 +-
 .../provider/AbstractClientProvider.java|  17 +-
 .../yarn/service/provider/ProviderUtils.java|  12 +-
 .../timelineservice/ServiceTimelineEvent.java   |   4 +-
 .../ServiceTimelinePublisher.java   |  48 +++---
 .../yarn/service/conf/TestAppJsonResolve.java   |   8 +-
 .../TestServiceTimelinePublisher.java   |   2 +-
 .../service/conf/examples/app-override.json |   6 +-
 .../markdown/yarn-service/Configurations.md | 165 +++
 .../src/site/markdown/yarn-service/Examples.md  |   4 +-
 .../src/site/markdown/yarn-service/Overview.md  |   3 +-
 .../markdown/yarn-service/YarnServiceAPI.md |  14 +-
 .../app/controllers/yarn-app/components.js  |   2 +-
 .../app/templates/components/metrics-table.hbs  |   2 +-
 .../templates/yarn-component-instance/info.hbs  |   4 +-
 .../templates/yarn-component-instances/info.hbs |   2 +-
 32 files changed, 384 insertions(+), 169 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6c4bd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
index 3cd3d48..00b21dd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -176,14 +176,14 @@ POST URL - 
http://localhost:9191:/ws/v1/services/hbase-app-1
   {
 "type": "XML",
 "dest_file": "/etc/hadoop/conf/core-site.xml",
-"props": {
+"properties": {
   "fs.defaultFS": "${CLUSTER_FS_URI}"
 }
   },
   {
 "type": "XML",
 "dest_file": "/etc/hbase/conf/hbase-site.xml",
-"props": {
+"properties": {
   "hbase.cluster.distributed": "true",
   "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
   "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",
@@ -216,14 +216,14 @@ POST URL - 
http://localhost:9191:/ws/v1/services/hbase-app-1
   {
 "type": "XML",
 "dest_file": "/etc/hadoop/conf/core-site.xml",
-"props": {
+"properties": {
   "fs.defaultFS": "${CLUSTER_FS_URI}"
 }
   },
   {
 "type": "XML",
 "dest_file": "/etc/hbase/conf/hbase-site.xml",
-"props": {
+"properties": {
   "hbase.cluster.distributed": "true",
   "hbase.zookeeper.quorum": "${CLUSTER_ZK_QUORUM}",
   "hbase.rootdir": "${SERVICE_HDFS_DIR}/hbase",

[09/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index c2866cf..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- * 
- * --instance {app name}, if  a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles 
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
-commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
-  public static final String USAGE =
-  "Usage: " + SliderActions.ACTION_REGISTRY
-  + " ("
-  + Arguments.ARG_LIST + "|"
-  + Arguments.ARG_LISTCONF + "|"
-  + Arguments.ARG_LISTEXP + "|"
-  + Arguments.ARG_LISTFILES + "|"
-  + Arguments.ARG_GETCONF + "|"
-  + Arguments.ARG_GETEXP + "> "
-  + Arguments.ARG_NAME + "  "
-  + " )"
-  + "[" + Arguments.ARG_VERBOSE + "] "
-  + "[" + Arguments.ARG_USER + "] "
-  + "[" + Arguments.ARG_OUTPUT + "  ] "
-  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
-  + "[" + Arguments.ARG_FORMAT + " ] "
-  + System.getProperty("line.separator")
-  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
-  ;
-  public ActionRegistryArgs() {
-  }
-
-  public ActionRegistryArgs(String name) {
-this.name = name;
-  }
-
-  @Override
-  public String getActionName() {
-return ACTION_REGISTRY;
-  }
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  @Override
-  public int getMinParams() {
-return 0;
-  }
-  
-  @Parameter(names = {ARG_LIST}, 
-  description = "list services")
-  public boolean list;
-
-  @Parameter(names = {ARG_LISTCONF}, 
-  description = "list configurations")
-  public boolean listConf;
-
-  @Parameter(names = {ARG_GETCONF},
-  description = "get configuration")
-  public String getConf;
-
-  @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
-  public boolean listExports;
-
-  @Parameter(names = {ARG_GETEXP},
- description = "get export")
-  public String getExport;
-
-  @Parameter(names = {ARG_LISTFILES},
-  description = "list files")
-  public String listFiles;
-
-  @Parameter(names = {ARG_GETFILES},
-  description = "get files")
-  public String getFiles;
-
-  //--format 
-  @Parameter(names = ARG_FORMAT,
-  description = "Format for a response: ")
-  public String format = ConfigFormat.XML.toString() ;
-
-  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
-  description = "Output destination")
-  public File out;
-
-  @Parameter(names 

[40/58] [abbrv] hadoop git commit: YARN-7191. Improve yarn-service documentation. Contributed by Jian He

[jianhe]

YARN-7191. Improve yarn-service documentation. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88512097
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88512097
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88512097

Branch: refs/heads/yarn-native-services
Commit: 8851209788a4e110e899dd28f628657a21981db5
Parents: c70e5df
Author: Billie Rinaldi 
Authored: Wed Sep 27 15:08:33 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:17 2017 -0800

--
 .../src/site/markdown/yarn-service/Concepts.md  |  47 +---
 .../src/site/markdown/yarn-service/Overview.md  |   3 +-
 .../site/markdown/yarn-service/QuickStart.md|  34 +--
 .../site/markdown/yarn-service/RegistryDNS.md   | 166 +
 .../markdown/yarn-service/ServiceDiscovery.md   | 235 ---
 5 files changed, 286 insertions(+), 199 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/88512097/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
index 7b62c36..e567d03 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -22,6 +22,8 @@ It also does all the heavy lifting work such as resolving the 
service definition
 failed containers, monitoring components' healthiness and readiness, ensuring 
dependency start order across components, flexing up/down components, 
 upgrading components etc. The end goal of the framework is to make sure the 
service is up and running as the state that user desired.
 
+In addition, it leverages a lot of features in YARN core to accomplish 
scheduling constraints, such as
+affinity and anti-affinity scheduling, log aggregation for services, 
automatically restart a container if it fails, and do in-place upgrade of a 
container.
 
 ### A Restful API-Server for deploying/managing services on YARN
 A restful API server is developed to allow users to deploy/manage their 
services on YARN via a simple JSON spec. This avoids users
@@ -34,44 +36,11 @@ support HA, distribute the load etc.
 
 ### Service Discovery
 A DNS server is implemented to enable discovering services on YARN via the 
standard mechanism: DNS lookup.
-The DNS server essentially exposes the information in YARN service registry by 
translating them into DNS records such as A record and SRV record.
-Clients can discover the IPs of containers via standard DNS lookup.
-The previous read mechanisms of YARN Service Registry were limited to a 
registry specific (java) API and a REST interface and are difficult
-to wireup existing clients and services. The DNS based service discovery 
eliminates this gap. Please refer to this [DNS doc](ServiceDiscovery.md) 
-for more details.
-
-### Scheduling
-
-A host of scheduling features are being developed to support long running 
services.
-
-* Affinity and anti-affinity scheduling across containers 
([YARN-6592](https://issues.apache.org/jira/browse/YARN-6592)).
-* Container resizing 
([YARN-1197](https://issues.apache.org/jira/browse/YARN-1197))
-* Special handling of container preemption/reservation for services 
-
-### Container auto-restarts
-
-[YARN-3998](https://issues.apache.org/jira/browse/YARN-3998) implements a 
retry-policy to let NM re-launch a service container when it fails.
-The service REST API provides users a way to enable NodeManager to 
automatically restart the container if it fails.
-The advantage is that it avoids the entire cycle of releasing the failed 
containers, re-asking new containers, re-do resource localizations and so on, 
which
-greatly minimizes container downtime.
 
+The framework posts container information such as hostname and ip into the 
[YARN service registry](../registry/index.md). And the DNS server essentially 
exposes the
+information in YARN service registry by translating them into DNS records such 
as A record and SRV record.
+Clients can then discover the IPs of containers via standard DNS lookup.
 
-### Container in-place upgrade
-
-[YARN-4726](https://issues.apache.org/jira/browse/YARN-4726) aims to support 
upgrading containers in-place, that is, without losing the container 
allocations.
-It opens up a few APIs in NodeManager to allow ApplicationMasters to upgrade 
their containers via a simple API call.
-Under the hood, NodeManager does below steps:
-* Downloading the new resources such as jars, 

[19/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 000..7baa284
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+  protected final List argumentList = new ArrayList<>(20);
+
+  /**
+   * Add an entry to the command list
+   * @param args arguments -these will be converted strings
+   */
+  public void add(Object... args) {
+for (Object arg : args) {
+  argumentList.add(arg.toString());
+}
+  }
+
+  // Get the number of arguments
+  public int size() {
+return argumentList.size();
+  }
+  
+  /**
+   * Append the output and error files to the tail of the command
+   * @param stdout out
+   * @param stderr error. Set this to null to append into stdout
+   */
+  public void addOutAndErrFiles(String stdout, String stderr) {
+Preconditions.checkNotNull(stdout, "Null output file");
+Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+// write out the path output
+argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+if (stderr != null) {
+  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
+   stderr);
+} else {
+  argumentList.add("2>&1");
+}
+  }
+
+  /**
+   * This just returns the command line
+   * @see #build()
+   * @return the command line
+   */
+  @Override
+  public String toString() {
+return build();
+  }
+
+  /**
+   * Build the command line
+   * @return the command line
+   */
+  public String build() {
+return SliderUtils.join(argumentList, " ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 000..0e51a62
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 

[49/58] [abbrv] hadoop git commit: YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He

[jianhe]

YARN-7351. Fix high CPU usage issue in RegistryDNS. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2775fbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2775fbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2775fbc

Branch: refs/heads/yarn-native-services
Commit: d2775fbccaa187729d86459542210434ce4c3b8b
Parents: 501be9b
Author: Billie Rinaldi 
Authored: Wed Oct 25 13:01:01 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:19 2017 -0800

--
 .../org/apache/hadoop/registry/server/dns/RegistryDNS.java   | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2775fbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 37e8429..9097c46 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -189,7 +189,8 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
 
 LOG.info("Opening TCP and UDP channels on {} port {}", addr, port);
 addNIOUDP(addr, port);
-addNIOTCP(addr, port);
+//TODO Fix dns lookup over TCP
+//addNIOTCP(addr, port);
   }
 
   /**
@@ -770,6 +771,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
   byte[] response = null;
   try {
 query = new Message(in);
+LOG.info("received TCP query {}", query.getQuestion());
 response = generateReply(query, ch.socket());
 if (response == null) {
   return;
@@ -947,7 +949,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
   input.flip();
   input.get(in);
   query = new Message(in);
-  LOG.info("{}:  received query {}", remoteAddress,
+  LOG.info("{}: received UDP query {}", remoteAddress,
   query.getQuestion());
   response = generateReply(query, null);
   if (response == null) {
@@ -960,7 +962,7 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
 output.put(response);
 output.flip();
 
-LOG.info("{}:  sending response", remoteAddress);
+LOG.debug("{}:  sending response", remoteAddress);
 channel.send(output, remoteAddress);
   }
 } catch (Exception e) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[13/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 000..b742553
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+  private TimelineV2Client timelineClient;
+  private Configuration config;
+  private ServiceTimelinePublisher serviceTimelinePublisher;
+  private static String SERVICE_NAME = "HBASE";
+  private static String SERVICEID = "application_1490093646524_0005";
+  private static String ARTIFACTID = "ARTIFACTID";
+  private static String COMPONENT_NAME = "DEFAULT";
+  private static String CONTAINER_ID =
+  "container_e02_1490093646524_0005_01_01";
+  private static String CONTAINER_IP =
+  "localhost";
+  private static String CONTAINER_HOSTNAME =
+  "cnl124-localhost.site";
+  private static String CONTAINER_BAREHOST =
+  "localhost.com";
+
+  @Before
+  public void setUp() throws Exception {
+config = new Configuration();
+config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+timelineClient =
+new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
+serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+timelineClient.init(config);
+serviceTimelinePublisher.init(config);
+timelineClient.start();
+serviceTimelinePublisher.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+if 

[17/58] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40ab068e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 000..add2475
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+  private static final Log LOG =
+  LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+  private final RegistryOperations registryOperations;
+  private final String user;
+  private final String sliderServiceClass;
+  private final String instanceName;
+  /**
+   * Record used where the service registered itself.
+   * Null until the service is registered
+   */
+  private ServiceRecord selfRegistration;
+
+  /**
+   * Path where record was registered
+   * Null until the service is registered
+   */
+  private String selfRegistrationPath;
+
+  public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+  String user,
+  String sliderServiceClass,
+  String instanceName,
+  ApplicationAttemptId applicationAttemptId) {
+Preconditions.checkArgument(registryOperations != null,
+"null registry operations");
+Preconditions.checkArgument(user != null, "null user");
+Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+"unset service class");
+Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+"instanceName");
+Preconditions.checkArgument(applicationAttemptId != null,
+"null applicationAttemptId");
+this.registryOperations = registryOperations;
+this.user = user;
+this.sliderServiceClass = sliderServiceClass;
+this.instanceName = instanceName;
+  }
+
+  public String getUser() {
+return user;
+  }
+
+
+  private void setSelfRegistration(ServiceRecord selfRegistration) {
+this.selfRegistration = selfRegistration;
+  }
+
+  /**
+   * Get the path to where the service has registered itself.
+   * Null until the service is registered
+   * @return the service registration path.
+   */
+  public String getSelfRegistrationPath() {
+return selfRegistrationPath;
+  }
+
+  /**
+   * Get the absolute path to where the service has registered itself.
+   * This includes the base registry path
+   * Null until the service is registered
+   * @return the service 

[52/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi

[jianhe]

YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba7ed7b6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba7ed7b6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba7ed7b6

Branch: refs/heads/yarn-native-services
Commit: ba7ed7b66a48c61086ea837f39ff0f22ffe31eca
Parents: 8ec9e72
Author: Jian He 
Authored: Sat Oct 28 23:04:19 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:19 2017 -0800

--
 NOTICE.txt  |   4 -
 hadoop-project/pom.xml  |   6 -
 hadoop-yarn-project/hadoop-yarn/bin/yarn|  27 +-
 .../hadoop-yarn/conf/yarn-env.sh|  18 +-
 .../hadoop-yarn-services-core/pom.xml   |   5 -
 .../hadoop/yarn/service/ServiceMaster.java  |  21 +-
 .../hadoop/yarn/service/client/ServiceCLI.java  | 113 ---
 .../yarn/service/client/ServiceClient.java  | 187 +--
 .../client/params/AbstractActionArgs.java   | 139 
 .../client/params/AbstractArgsDelegate.java |  28 --
 .../AbstractClusterBuildingActionArgs.java  |  46 ---
 .../service/client/params/ActionBuildArgs.java  |  39 ---
 .../service/client/params/ActionClientArgs.java |  71 -
 .../service/client/params/ActionCreateArgs.java |  46 ---
 .../client/params/ActionDependencyArgs.java |  65 
 .../client/params/ActionDestroyArgs.java|  37 ---
 .../service/client/params/ActionExistsArgs.java |  49 ---
 .../service/client/params/ActionFlexArgs.java   |  50 ---
 .../service/client/params/ActionFreezeArgs.java |  56 
 .../service/client/params/ActionHelpArgs.java   |  44 ---
 .../service/client/params/ActionKeytabArgs.java |  76 -
 .../service/client/params/ActionListArgs.java   |  76 -
 .../client/params/ActionRegistryArgs.java   | 218 -
 .../client/params/ActionResolveArgs.java| 153 -
 .../client/params/ActionResourceArgs.java   |  70 
 .../service/client/params/ActionStatusArgs.java |  51 ---
 .../service/client/params/ActionThawArgs.java   |  67 
 .../service/client/params/ActionTokensArgs.java |  78 -
 .../service/client/params/ActionUpdateArgs.java |  37 ---
 .../yarn/service/client/params/ArgOps.java  | 156 -
 .../yarn/service/client/params/Arguments.java   | 103 --
 .../yarn/service/client/params/ClientArgs.java  | 193 ---
 .../yarn/service/client/params/CommonArgs.java  | 245 --
 .../client/params/ComponentArgsDelegate.java|  52 ---
 .../client/params/DontSplitArguments.java   |  34 --
 .../client/params/LaunchArgsAccessor.java   |  30 --
 .../client/params/LaunchArgsDelegate.java   |  51 ---
 .../client/params/OptionArgsDelegate.java   |  66 
 .../client/params/PathArgumentConverter.java|  34 --
 .../service/client/params/ServiceAMArgs.java|  50 ---
 .../client/params/ServiceAMCreateAction.java|  37 ---
 .../service/client/params/SliderActions.java|  79 -
 .../service/client/params/WaitArgsDelegate.java |  42 ---
 .../service/client/params/WaitTimeAccessor.java |  24 --
 .../hadoop/yarn/service/utils/SliderUtils.java  |   6 +-
 .../client/TestBuildExternalComponents.java |  19 +-
 .../yarn/service/client/TestServiceCLI.java | 125 +++-
 .../hadoop/yarn/client/api/AppAdminClient.java  | 222 +
 .../hadoop/yarn/client/cli/ApplicationCLI.java  | 318 ---
 .../hadoop/yarn/client/cli/TestYarnCLI.java | 158 ++---
 .../src/site/markdown/YarnCommands.md   |  86 ++---
 .../src/site/markdown/yarn-service/Examples.md  |  12 +-
 .../site/markdown/yarn-service/QuickStart.md|  46 ++-
 .../markdown/yarn-service/YarnServiceAPI.md |  20 +-
 54 files changed, 840 insertions(+), 3245 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index f3af2f7..bc7a26f 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -582,10 +582,6 @@ Ehcache 3.3.1,
 which has the following notices:
  * Ehcache V3 Copyright 2014-2016 Terracotta, Inc.
 
-JCommander (https://github.com/cbeust/jcommander),
-which has the following notices:
- * Copyright 2010 Cedric Beust ced...@beust.com
-
 The binary distribution of this product bundles binaries of
 snakeyaml (https://bitbucket.org/asomov/snakeyaml),
 which has the following notices:

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 

[50/58] [abbrv] hadoop git commit: YARN-7384. Remove apiserver cmd and merge service cmd into application cmd. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba7ed7b6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
index f22d487..df4b1df 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceCLI.java
@@ -20,88 +20,71 @@ package org.apache.hadoop.yarn.service.client;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
 import org.apache.hadoop.yarn.service.api.records.Component;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
+import org.apache.hadoop.yarn.service.api.records.Service;
 import org.apache.hadoop.yarn.service.conf.ExampleAppJson;
 import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.apache.hadoop.yarn.util.Records;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 
-import static 
org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS;
-import static 
org.apache.hadoop.yarn.conf.YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS;
-import static org.apache.hadoop.yarn.service.client.params.Arguments.ARG_FILE;
 import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_BASE_PATH;
-import static org.mockito.Mockito.*;
 
 public class TestServiceCLI {
+  private static final Logger LOG = LoggerFactory.getLogger(TestServiceCLI
+  .class);
 
-  protected Configuration conf = new YarnConfiguration();
+  private Configuration conf = new YarnConfiguration();
   private File basedir;
-  private ServiceCLI cli;
   private SliderFileSystem fs;
+  private String basedirProp;
 
-  private void buildApp(String appDef) throws Throwable {
-String[] args =
-{ "build", ARG_FILE, ExampleAppJson.resourceName(appDef)};
-ClientArgs clientArgs = new ClientArgs(args);
-clientArgs.parse();
-cli.exec(clientArgs);
+  private void runCLI(String[] args) throws Exception {
+LOG.info("running CLI: yarn {}", Arrays.asList(args));
+ApplicationCLI cli = new ApplicationCLI();
+cli.setSysOutPrintStream(System.out);
+cli.setSysErrPrintStream(System.err);
+int res = ToolRunner.run(cli, ApplicationCLI.preProcessArgs(args));
+cli.stop();
+  }
+
+  private void buildApp(String serviceName, String appDef) throws Throwable {
+String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ExampleAppJson.resourceName(appDef)};
+runCLI(args);
+  }
+
+  private void buildApp(String serviceName, String appDef, String lifetime,
+  String queue) throws Throwable {
+String[] args = {"app", "-D", basedirProp, "-save", serviceName,
+ExampleAppJson.resourceName(appDef), "-updateLifetime", lifetime,
+"-changeQueue", queue};
+runCLI(args);
   }
 
   @Before
   public void setup() throws Throwable {
 basedir = new File("target", "apps");
+basedirProp = YARN_SERVICE_BASE_PATH + "=" + basedir.getAbsolutePath();
 conf.set(YARN_SERVICE_BASE_PATH, basedir.getAbsolutePath());
-conf.setLong(RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 0);
-conf.setLong(RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1);
-conf.setInt(
-CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
-conf.setInt(CommonConfigurationKeysPublic.
-IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY, 0);
 fs = new SliderFileSystem(conf);
 if (basedir.exists()) {
   FileUtils.deleteDirectory(basedir);
 } else {
   basedir.mkdirs();
 }
-
-// create a CLI and skip 

[36/58] [abbrv] hadoop git commit: YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He

[jianhe]

YARN-7210. Some NPE fixes in Registry DNS. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce74e643
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce74e643
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce74e643

Branch: refs/heads/yarn-native-services
Commit: ce74e64363abc64561263be70a923ab3e67f043f
Parents: 37c9b73
Author: Billie Rinaldi 
Authored: Thu Sep 21 10:18:42 2017 -0700
Committer: Jian He 
Committed: Mon Nov 6 13:30:16 2017 -0800

--
 .../hadoop/yarn/service/ServiceScheduler.java |  6 +-
 .../hadoop/yarn/service/client/ServiceClient.java |  3 +++
 .../params/AbstractClusterBuildingActionArgs.java |  2 +-
 .../yarn/service/client/params/Arguments.java |  1 +
 .../component/instance/ComponentInstance.java | 10 +-
 .../service/provider/AbstractProviderService.java | 18 +++---
 .../client/binding/RegistryTypeUtils.java |  3 ++-
 .../dns/ApplicationServiceRecordProcessor.java| 15 ++-
 .../server/dns/BaseServiceRecordProcessor.java|  8 
 .../registry/server/dns/TestRegistryDNS.java  | 12 
 .../runtime/DockerLinuxContainerRuntime.java  |  4 ++--
 .../src/site/markdown/YarnCommands.md |  1 +
 12 files changed, 53 insertions(+), 30 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 7b809b9..ec5f3ed 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -344,11 +344,7 @@ public class ServiceScheduler extends CompositeService {
 attemptId.getApplicationId().toString());
 serviceRecord.set(YarnRegistryAttributes.YARN_PERSISTENCE,
 PersistencePolicies.APPLICATION);
-serviceRecord.description = "Yarn Service Master";
-
-serviceRecord.addExternalEndpoint(RegistryTypeUtils
-.ipcEndpoint("classpath:org.apache.hadoop.yarn.service.appmaster.ipc",
-context.clientAMService.getBindAddress()));
+serviceRecord.description = "YarnServiceMaster";
 
 // set any provided attributes
 setUserProvidedServiceRecordAttributes(service.getConfiguration(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 6890aef..a3a9fd0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -170,6 +170,9 @@ public class ServiceClient extends CompositeService
 if (!StringUtils.isEmpty(args.getServiceName())) {
   service.setName(args.getServiceName());
 }
+if (!StringUtils.isEmpty(args.queue)) {
+  service.setQueue(args.queue);
+}
 return service;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce74e643/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/AbstractClusterBuildingActionArgs.java
--
diff --git