hadoop git commit: HADOOP-13706. Update jackson from 1.9.13 to 2.x in hadoop-common-project.

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 47ca9e26f -> 67d9f2808


HADOOP-13706. Update jackson from 1.9.13 to 2.x in hadoop-common-project.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67d9f280
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67d9f280
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67d9f280

Branch: refs/heads/trunk
Commit: 67d9f2808efb34b9a7b0b824cb4033b95ad33474
Parents: 47ca9e2
Author: Akira Ajisaka 
Authored: Tue Nov 29 14:07:19 2016 +0900
Committer: Akira Ajisaka 
Committed: Tue Nov 29 14:07:19 2016 +0900

--
 hadoop-common-project/hadoop-common/pom.xml   | 14 --
 .../java/org/apache/hadoop/conf/Configuration.java|  8 
 .../hadoop/crypto/key/kms/KMSClientProvider.java  |  2 +-
 .../main/java/org/apache/hadoop/fs/shell/Display.java | 11 +--
 .../java/org/apache/hadoop/ipc/DecayRpcScheduler.java |  2 +-
 .../src/main/java/org/apache/hadoop/ipc/Server.java   |  2 +-
 .../java/org/apache/hadoop/jmx/JMXJsonServlet.java|  6 +++---
 .../src/main/java/org/apache/hadoop/log/Log4Json.java | 14 +++---
 .../web/DelegationTokenAuthenticationHandler.java |  6 +++---
 .../delegation/web/DelegationTokenAuthenticator.java  |  2 +-
 .../org/apache/hadoop/util/HttpExceptionUtils.java|  2 +-
 .../org/apache/hadoop/conf/TestConfiguration.java |  2 +-
 .../test/java/org/apache/hadoop/log/TestLog4Json.java | 13 +
 ...DelegationTokenAuthenticationHandlerWithMocks.java |  3 ++-
 .../token/delegation/web/TestWebDelegationToken.java  |  4 +---
 .../apache/hadoop/util/TestHttpExceptionUtils.java|  2 +-
 hadoop-common-project/hadoop-kms/pom.xml  |  4 
 .../hadoop/crypto/key/kms/server/KMSJSONReader.java   |  3 ++-
 .../hadoop/crypto/key/kms/server/KMSJSONWriter.java   |  3 ++-
 19 files changed, 45 insertions(+), 58 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml 
b/hadoop-common-project/hadoop-common/pom.xml
index f442b2c4..1d45452 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -192,16 +192,6 @@
   runtime
 
 
-  org.codehaus.jackson
-  jackson-core-asl
-  compile
-
-
-  org.codehaus.jackson
-  jackson-mapper-asl
-  compile
-
-
   org.mockito
   mockito-all
   test
@@ -324,6 +314,10 @@
   org.apache.kerby
   kerb-simplekdc
 
+
+  com.fasterxml.jackson.core
+  jackson-databind
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/67d9f280/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index dbbc8ff..c0e42e5 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -18,6 +18,8 @@
 
 package org.apache.hadoop.conf;
 
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
 import com.google.common.annotations.VisibleForTesting;
 
 import java.io.BufferedInputStream;
@@ -91,8 +93,6 @@ import 
org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringInterner;
 import org.apache.hadoop.util.StringUtils;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
 import org.w3c.dom.Attr;
 import org.w3c.dom.DOMException;
 import org.w3c.dom.Document;
@@ -3028,7 +3028,7 @@ public class Configuration implements 
Iterable>,
   propertyName + " not found");
 } else {
   JsonFactory dumpFactory = new JsonFactory();
-  JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
+  JsonGenerator dumpGenerator = dumpFactory.createGenerator(out);
   dumpGenerator.writeStartObject();
   dumpGenerator.writeFieldName("property");
   appendJSONProperty(dumpGenerator, config, propertyName);
@@ -3066,7 +3066,7 @@ public class Configuration implements 
Iterable>,
   public static void dumpConfiguration(Configuration config,
   Writer out) throws IOException {
 JsonFactory 

hadoop git commit: HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 59f606756 -> 92ee5e207


HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)

(cherry picked from commit 47ca9e26fba4a639e43bee5bfc001ffc4b42330d)
(cherry picked from commit 285edf691db42a3e17138d960f8560c62d8d0dde)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92ee5e20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92ee5e20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92ee5e20

Branch: refs/heads/branch-2.8
Commit: 92ee5e2077c4a1b2d2795dc4f0afbf81dea374f3
Parents: 59f6067
Author: Robert Kanter 
Authored: Mon Nov 28 18:08:09 2016 -0800
Committer: Xiao Chen 
Committed: Mon Nov 28 18:23:23 2016 -0800

--
 .../crypto/key/kms/KMSClientProvider.java   | 41 ++---
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 97 +++-
 .../hdfs/server/namenode/FSNamesystem.java  |  7 ++
 3 files changed, 110 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92ee5e20/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 145fd62..3ce7f1d 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -173,14 +173,20 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   LOG.debug("Renewing delegation token {}", token);
   KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
   KeyProviderFactory.KEY_PROVIDER_PATH);
-  if (!(keyProvider instanceof
-  KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
-"null" : keyProvider.getClass());
-return 0;
+  try {
+if (!(keyProvider instanceof
+KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+  LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
+  "null" : keyProvider.getClass());
+  return 0;
+}
+return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+keyProvider).renewDelegationToken(token);
+  } finally {
+if (keyProvider != null) {
+  keyProvider.close();
+}
   }
-  return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-  keyProvider).renewDelegationToken(token);
 }
 
 @Override
@@ -188,14 +194,20 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   LOG.debug("Canceling delegation token {}", token);
   KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
   KeyProviderFactory.KEY_PROVIDER_PATH);
-  if (!(keyProvider instanceof
-  KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
-"null" : keyProvider.getClass());
-return;
+  try {
+if (!(keyProvider instanceof
+KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+  LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
+  "null" : keyProvider.getClass());
+  return;
+}
+((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+keyProvider).cancelDelegationToken(token);
+  } finally {
+if (keyProvider != null) {
+  keyProvider.close();
+}
   }
-  ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-  keyProvider).cancelDelegationToken(token);
 }
   }
 
@@ -1072,6 +1084,7 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
 } finally {
   if (sslFactory != null) {
 sslFactory.destroy();
+sslFactory = null;
   }
 }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/92ee5e20/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git 
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
 

hadoop git commit: HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)

[rkanter]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 1b831d8e0 -> 285edf691


HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)

(cherry picked from commit 47ca9e26fba4a639e43bee5bfc001ffc4b42330d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/285edf69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/285edf69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/285edf69

Branch: refs/heads/branch-2
Commit: 285edf691db42a3e17138d960f8560c62d8d0dde
Parents: 1b831d8
Author: Robert Kanter 
Authored: Mon Nov 28 18:08:09 2016 -0800
Committer: Robert Kanter 
Committed: Mon Nov 28 18:12:53 2016 -0800

--
 .../crypto/key/kms/KMSClientProvider.java   | 41 ++---
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 97 +++-
 .../hdfs/server/namenode/FSNamesystem.java  |  7 ++
 3 files changed, 110 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/285edf69/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 145fd62..3ce7f1d 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -173,14 +173,20 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   LOG.debug("Renewing delegation token {}", token);
   KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
   KeyProviderFactory.KEY_PROVIDER_PATH);
-  if (!(keyProvider instanceof
-  KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
-"null" : keyProvider.getClass());
-return 0;
+  try {
+if (!(keyProvider instanceof
+KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+  LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
+  "null" : keyProvider.getClass());
+  return 0;
+}
+return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+keyProvider).renewDelegationToken(token);
+  } finally {
+if (keyProvider != null) {
+  keyProvider.close();
+}
   }
-  return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-  keyProvider).renewDelegationToken(token);
 }
 
 @Override
@@ -188,14 +194,20 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   LOG.debug("Canceling delegation token {}", token);
   KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
   KeyProviderFactory.KEY_PROVIDER_PATH);
-  if (!(keyProvider instanceof
-  KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
-"null" : keyProvider.getClass());
-return;
+  try {
+if (!(keyProvider instanceof
+KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+  LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
+  "null" : keyProvider.getClass());
+  return;
+}
+((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+keyProvider).cancelDelegationToken(token);
+  } finally {
+if (keyProvider != null) {
+  keyProvider.close();
+}
   }
-  ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-  keyProvider).cancelDelegationToken(token);
 }
   }
 
@@ -1072,6 +1084,7 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
 } finally {
   if (sslFactory != null) {
 sslFactory.destroy();
+sslFactory = null;
   }
 }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/285edf69/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git 
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
 
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index edb4cfc..7a6371b 100644
--- 

hadoop git commit: HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)

[rkanter]

Repository: hadoop
Updated Branches:
  refs/heads/trunk d60a60be8 -> 47ca9e26f


HADOOP-13838. KMSTokenRenewer should close providers (xiaochen via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47ca9e26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47ca9e26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47ca9e26

Branch: refs/heads/trunk
Commit: 47ca9e26fba4a639e43bee5bfc001ffc4b42330d
Parents: d60a60b
Author: Robert Kanter 
Authored: Mon Nov 28 18:08:09 2016 -0800
Committer: Robert Kanter 
Committed: Mon Nov 28 18:08:09 2016 -0800

--
 .../crypto/key/kms/KMSClientProvider.java   | 41 ++---
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 97 +++-
 .../hdfs/server/namenode/FSNamesystem.java  |  7 ++
 3 files changed, 110 insertions(+), 35 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47ca9e26/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 2b6ae9e..f184ea7 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -173,14 +173,20 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   LOG.debug("Renewing delegation token {}", token);
   KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
   KeyProviderFactory.KEY_PROVIDER_PATH);
-  if (!(keyProvider instanceof
-  KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
-"null" : keyProvider.getClass());
-return 0;
+  try {
+if (!(keyProvider instanceof
+KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+  LOG.warn("keyProvider {} cannot renew dt.", keyProvider == null ?
+  "null" : keyProvider.getClass());
+  return 0;
+}
+return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+keyProvider).renewDelegationToken(token);
+  } finally {
+if (keyProvider != null) {
+  keyProvider.close();
+}
   }
-  return ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-  keyProvider).renewDelegationToken(token);
 }
 
 @Override
@@ -188,14 +194,20 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
   LOG.debug("Canceling delegation token {}", token);
   KeyProvider keyProvider = KMSUtil.createKeyProvider(conf,
   KeyProviderFactory.KEY_PROVIDER_PATH);
-  if (!(keyProvider instanceof
-  KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
-LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
-"null" : keyProvider.getClass());
-return;
+  try {
+if (!(keyProvider instanceof
+KeyProviderDelegationTokenExtension.DelegationTokenExtension)) {
+  LOG.warn("keyProvider {} cannot cancel dt.", keyProvider == null ?
+  "null" : keyProvider.getClass());
+  return;
+}
+((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
+keyProvider).cancelDelegationToken(token);
+  } finally {
+if (keyProvider != null) {
+  keyProvider.close();
+}
   }
-  ((KeyProviderDelegationTokenExtension.DelegationTokenExtension)
-  keyProvider).cancelDelegationToken(token);
 }
   }
 
@@ -1072,6 +1084,7 @@ public class KMSClientProvider extends KeyProvider 
implements CryptoExtension,
 } finally {
   if (sslFactory != null) {
 sslFactory.destroy();
+sslFactory = null;
   }
 }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/47ca9e26/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git 
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
 
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index dac91e0..aaab172 100644
--- 

[2/3] hadoop git commit: HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve Loughran

[liuml07]

HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve 
Loughran

(cherry picked from commit d60a60be8aa450c44d3be69d26c88025e253ac0c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b831d8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b831d8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b831d8e

Branch: refs/heads/branch-2
Commit: 1b831d8e01ab46499584b82d3d1a7fae951ae909
Parents: 1c0fa25
Author: Mingliang Liu 
Authored: Mon Nov 28 16:30:29 2016 -0800
Committer: Mingliang Liu 
Committed: Mon Nov 28 16:35:18 2016 -0800

--
 .../hadoop/fs/s3a/RenameFailedException.java| 70 +++
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +---
 .../fs/s3a/ITestS3AFileSystemContract.java  | 15 
 .../src/test/resources/contract/s3a.xml |  2 +-
 4 files changed, 130 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b831d8e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
new file mode 100644
index 000..6c88f3b
--- /dev/null
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Error to indicate that a specific rename failed.
+ * The exit code defines the exit code to be returned in the {@code rename()}
+ * call.
+ * Target path is set to destination.
+ */
+public class RenameFailedException extends PathIOException {
+
+  /**
+   * Exit code to be returned.
+   */
+  private boolean exitCode = false;
+
+  public RenameFailedException(String src, String dest, Throwable cause) {
+super(src, cause);
+setOperation("rename");
+setTargetPath(dest);
+  }
+
+  public RenameFailedException(String src, String dest, String error) {
+super(src, error);
+setOperation("rename");
+setTargetPath(dest);
+  }
+
+  public RenameFailedException(Path src, Path optionalDest, String error) {
+super(src.toString(), error);
+setOperation("rename");
+if (optionalDest != null) {
+  setTargetPath(optionalDest.toString());
+}
+  }
+
+  public boolean getExitCode() {
+return exitCode;
+  }
+
+  /**
+   * Set the exit code.
+   * @param code exit code to raise
+   * @return the exception
+   */
+  public RenameFailedException withExitCode(boolean code) {
+this.exitCode = code;
+return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b831d8e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index b9b8810..5ac18a9 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -628,10 +628,12 @@ public class S3AFileSystem extends FileSystem {
* there is no Progressable passed in, this can time out jobs.
*
* Note: This implementation differs with other S3 drivers. Specifically:
+   * 
*   Fails if src is a file and dst is a directory.
*   Fails if src is a directory and dst is a file.
*   Fails if the parent of dst does not exist or is a file.
*   Fails if dst is a directory that is not empty.
+   * 
*
* @param src path to be renamed
* @param dst 

[3/3] hadoop git commit: HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve Loughran

[liuml07]

HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve 
Loughran

(cherry picked from commit d60a60be8aa450c44d3be69d26c88025e253ac0c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59f60675
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59f60675
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59f60675

Branch: refs/heads/branch-2.8
Commit: 59f60675687c3251372392e9f522bc22d3c7ceaa
Parents: 713d800
Author: Mingliang Liu 
Authored: Mon Nov 28 16:30:29 2016 -0800
Committer: Mingliang Liu 
Committed: Mon Nov 28 16:42:41 2016 -0800

--
 .../hadoop/fs/s3a/RenameFailedException.java| 70 +++
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +---
 .../fs/s3a/ITestS3AFileSystemContract.java  | 15 
 .../src/test/resources/contract/s3a.xml |  2 +-
 4 files changed, 130 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/59f60675/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
new file mode 100644
index 000..6c88f3b
--- /dev/null
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Error to indicate that a specific rename failed.
+ * The exit code defines the exit code to be returned in the {@code rename()}
+ * call.
+ * Target path is set to destination.
+ */
+public class RenameFailedException extends PathIOException {
+
+  /**
+   * Exit code to be returned.
+   */
+  private boolean exitCode = false;
+
+  public RenameFailedException(String src, String dest, Throwable cause) {
+super(src, cause);
+setOperation("rename");
+setTargetPath(dest);
+  }
+
+  public RenameFailedException(String src, String dest, String error) {
+super(src, error);
+setOperation("rename");
+setTargetPath(dest);
+  }
+
+  public RenameFailedException(Path src, Path optionalDest, String error) {
+super(src.toString(), error);
+setOperation("rename");
+if (optionalDest != null) {
+  setTargetPath(optionalDest.toString());
+}
+  }
+
+  public boolean getExitCode() {
+return exitCode;
+  }
+
+  /**
+   * Set the exit code.
+   * @param code exit code to raise
+   * @return the exception
+   */
+  public RenameFailedException withExitCode(boolean code) {
+this.exitCode = code;
+return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/59f60675/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index b9b8810..5ac18a9 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -628,10 +628,12 @@ public class S3AFileSystem extends FileSystem {
* there is no Progressable passed in, this can time out jobs.
*
* Note: This implementation differs with other S3 drivers. Specifically:
+   * 
*   Fails if src is a file and dst is a directory.
*   Fails if src is a directory and dst is a file.
*   Fails if the parent of dst does not exist or is a file.
*   Fails if dst is a directory that is not empty.
+   * 
*
* @param src path to be renamed
* @param dst 

[1/3] hadoop git commit: HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve Loughran

[liuml07]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 1c0fa25dc -> 1b831d8e0
  refs/heads/branch-2.8 713d800e7 -> 59f606756
  refs/heads/trunk be88d574a -> d60a60be8


HADOOP-13823. s3a rename: fail if dest file exists. Contributed by Steve 
Loughran


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d60a60be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d60a60be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d60a60be

Branch: refs/heads/trunk
Commit: d60a60be8aa450c44d3be69d26c88025e253ac0c
Parents: be88d57
Author: Mingliang Liu 
Authored: Mon Nov 28 16:30:29 2016 -0800
Committer: Mingliang Liu 
Committed: Mon Nov 28 16:30:29 2016 -0800

--
 .../hadoop/fs/s3a/RenameFailedException.java| 70 +++
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +---
 .../fs/s3a/ITestS3AFileSystemContract.java  | 15 
 .../src/test/resources/contract/s3a.xml |  2 +-
 4 files changed, 130 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d60a60be/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
new file mode 100644
index 000..6c88f3b
--- /dev/null
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/RenameFailedException.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Error to indicate that a specific rename failed.
+ * The exit code defines the exit code to be returned in the {@code rename()}
+ * call.
+ * Target path is set to destination.
+ */
+public class RenameFailedException extends PathIOException {
+
+  /**
+   * Exit code to be returned.
+   */
+  private boolean exitCode = false;
+
+  public RenameFailedException(String src, String dest, Throwable cause) {
+super(src, cause);
+setOperation("rename");
+setTargetPath(dest);
+  }
+
+  public RenameFailedException(String src, String dest, String error) {
+super(src, error);
+setOperation("rename");
+setTargetPath(dest);
+  }
+
+  public RenameFailedException(Path src, Path optionalDest, String error) {
+super(src.toString(), error);
+setOperation("rename");
+if (optionalDest != null) {
+  setTargetPath(optionalDest.toString());
+}
+  }
+
+  public boolean getExitCode() {
+return exitCode;
+  }
+
+  /**
+   * Set the exit code.
+   * @param code exit code to raise
+   * @return the exception
+   */
+  public RenameFailedException withExitCode(boolean code) {
+this.exitCode = code;
+return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d60a60be/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 925abbc..f1cde4e 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -630,10 +630,12 @@ public class S3AFileSystem extends FileSystem {
* there is no Progressable passed in, this can time out jobs.
*
* Note: This implementation differs with other S3 drivers. Specifically:
+   * 
*   Fails if src is a file and dst is a directory.
*   Fails if src is a directory and dst is a file.
*   Fails if the parent of dst does not exist or is a file.
*   Fails if dst 

hadoop git commit: HADOOP-13742. Expose NumOpenConnectionsPerUser as a metric. Brahma Reddy Battula.

[zhz]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 c594904ec -> ba10a055e


HADOOP-13742. Expose NumOpenConnectionsPerUser as a metric. Brahma Reddy 
Battula.

(cherry picked from commit bd3735554fa5c3bc064c57ec78f4308430b14b48)
(cherry picked from commit 0fb1306f9329be7bc793a342d08b2fe16f4a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba10a055
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba10a055
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba10a055

Branch: refs/heads/branch-2.7
Commit: ba10a055e4c48ba10cd3e04136560d80f8020edc
Parents: c594904
Author: Kihwal Lee 
Authored: Thu Nov 17 12:20:01 2016 -0600
Committer: Zhe Zhang 
Committed: Mon Nov 28 15:37:45 2016 -0800

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +
 .../main/java/org/apache/hadoop/ipc/Server.java | 61 +++-
 .../apache/hadoop/ipc/metrics/RpcMetrics.java   |  5 ++
 .../java/org/apache/hadoop/ipc/TestRPC.java | 26 +
 .../org/apache/hadoop/test/MetricsAsserts.java  |  7 +++
 5 files changed, 100 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba10a055/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d9c930e..707c90e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -24,6 +24,9 @@ Release 2.7.4 - UNRELEASED
 HADOOP-13655. document object store use with fs shell and distcp.
 (Steve Loughran via Mingliang Liu)
 
+HADOOP-13742. Expose "NumOpenConnectionsPerUser" as a metric.
+(Brahma Reddy Battula via kihwal)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba10a055/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index d0e5022..487fd21 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -127,6 +127,7 @@ import com.google.protobuf.ByteString;
 import com.google.protobuf.CodedOutputStream;
 import com.google.protobuf.Message;
 import com.google.protobuf.Message.Builder;
+import org.codehaus.jackson.map.ObjectMapper;
 
 /** An abstract IPC service.  IPC calls take a single {@link Writable} as a
  * parameter, and return a {@link Writable} as their value.  A service runs on
@@ -1837,6 +1838,9 @@ public abstract class Server {
   authorizeConnection();
   // don't set until after authz because connection isn't established
   connectionContextRead = true;
+  if (user != null) {
+connectionManager.incrUserConnections(user.getShortUserName());
+  }
 }
 
 /**
@@ -2690,7 +2694,20 @@ public abstract class Server {
   public int getNumOpenConnections() {
 return connectionManager.size();
   }
-  
+
+  /**
+   * Get the NumOpenConnections/User.
+   */
+  public String getNumOpenConnectionsPerUser() {
+ObjectMapper mapper = new ObjectMapper();
+try {
+  return mapper
+  .writeValueAsString(connectionManager.getUserToConnectionsMap());
+} catch (IOException ignored) {
+}
+return null;
+  }
+
   /**
* The number of rpc calls in the queue.
* @return The number of rpc calls in the queue.
@@ -2802,6 +2819,9 @@ public abstract class Server {
   private class ConnectionManager {
 final private AtomicInteger count = new AtomicInteger();
 final private Set connections;
+/* Map to maintain the statistics per User */
+final private Map userToConnectionsMap;
+final private Object userToConnectionsMapLock = new Object();
 
 final private Timer idleScanTimer;
 final private int idleScanThreshold;
@@ -2833,6 +2853,7 @@ public abstract class Server {
   this.connections = Collections.newSetFromMap(
   new ConcurrentHashMap(
   maxQueueSize, 0.75f, readThreads+2));
+  this.userToConnectionsMap = new ConcurrentHashMap<>();
 }
 
 private boolean add(Connection connection) {
@@ -2850,7 +2871,39 @@ public abstract class Server {
   }
   return removed;
 }
-
+
+void incrUserConnections(String user) {
+  synchronized (userToConnectionsMapLock) {
+Integer count = 

[1/2] hadoop git commit: MAPREDUCE-6743. nativetask unit tests need to provide usable output; fix link errors during mvn test

[aw]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 62b42ef5d -> be88d574a


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
index 4f3804f..c04205d 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/include/gtest/gtest.h
@@ -126,8 +126,11 @@
 // Authors: w...@google.com (Zhanyong Wan)
 //
 // Low-level types and utilities for porting Google Test to various
-// platforms.  They are subject to change without notice.  DO NOT USE
-// THEM IN USER CODE.
+// platforms.  All macros ending with _ and symbols defined in an
+// internal namespace are subject to change without notice.  Code
+// outside Google Test MUST NOT USE THEM DIRECTLY.  Macros that don't
+// end with _ are part of Google Test's public API and can be used by
+// code outside Google Test.
 //
 // This file is fundamental to Google Test.  All other Google Test source
 // files are expected to #include this.  Therefore, it cannot #include
@@ -136,9 +139,30 @@
 #ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
 #define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_
 
-// The user can define the following macros in the build script to
-// control Google Test's behavior.  If the user doesn't define a macro
-// in this list, Google Test will define it.
+// Environment-describing macros
+// -
+//
+// Google Test can be used in many different environments.  Macros in
+// this section tell Google Test what kind of environment it is being
+// used in, such that Google Test can provide environment-specific
+// features and implementations.
+//
+// Google Test tries to automatically detect the properties of its
+// environment, so users usually don't need to worry about these
+// macros.  However, the automatic detection is not perfect.
+// Sometimes it's necessary for a user to define some of the following
+// macros in the build script to override Google Test's decisions.
+//
+// If the user doesn't define a macro in the list, Google Test will
+// provide a default definition.  After this header is #included, all
+// macros in this list will be defined to either 1 or 0.
+//
+// Notes to maintainers:
+//   - Each macro here is a user-tweakable knob; do not grow the list
+// lightly.
+//   - Use #if to key off these macros.  Don't use #ifdef or "#if
+// defined(...)", which will not work as these macros are ALWAYS
+// defined.
 //
 //   GTEST_HAS_CLONE  - Define it to 1/0 to indicate that clone(2)
 //  is/isn't available.
@@ -182,18 +206,23 @@
 //- Define to 1 when compiling Google Test itself
 //  as a shared library.
 
-// This header defines the following utilities:
+// Platform-indicating macros
+// --
+//
+// Macros indicating the platform on which Google Test is being used
+// (a macro is defined to 1 if compiled on the given platform;
+// otherwise UNDEFINED -- it's never defined to 0.).  Google Test
+// defines these macros automatically.  Code outside Google Test MUST
+// NOT define them.
 //
-// Macros indicating the current platform (defined to 1 if compiled on
-// the given platform; otherwise undefined):
 //   GTEST_OS_AIX  - IBM AIX
 //   GTEST_OS_CYGWIN   - Cygwin
+//   GTEST_OS_FREEBSD  - FreeBSD
 //   GTEST_OS_HPUX - HP-UX
 //   GTEST_OS_LINUX- Linux
 // GTEST_OS_LINUX_ANDROID - Google Android
 //   GTEST_OS_MAC  - Mac OS X
 // GTEST_OS_IOS- iOS
-//   GTEST_OS_IOS_SIMULATOR - iOS simulator
 //   GTEST_OS_NACL - Google Native Client (NaCl)
 //   GTEST_OS_OPENBSD  - OpenBSD
 //   GTEST_OS_QNX  - QNX
@@ -203,6 +232,8 @@
 // GTEST_OS_WINDOWS_DESKTOP  - Windows Desktop
 // GTEST_OS_WINDOWS_MINGW- MinGW
 // GTEST_OS_WINDOWS_MOBILE   - Windows Mobile
+// GTEST_OS_WINDOWS_PHONE- Windows Phone
+// GTEST_OS_WINDOWS_RT   - Windows Store App/WinRT
 //   GTEST_OS_ZOS  - z/OS
 //
 // Among the platforms, Cygwin, Linux, Max OS X, and Windows have the
@@ -212,22 +243,50 @@
 // googletestframew...@googlegroups.com (patches for fixing them are
 // even more welcome!).
 //
-// Note that it is possible that none of the GTEST_OS_* macros are defined.
+// It is possible that none of the GTEST_OS_* macros are defined.
+
+// Feature-indicating macros
+// 

[2/2] hadoop git commit: MAPREDUCE-6743. nativetask unit tests need to provide usable output; fix link errors during mvn test

[aw]

MAPREDUCE-6743. nativetask unit tests need to provide usable output; fix link 
errors during mvn test


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be88d574
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be88d574
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be88d574

Branch: refs/heads/trunk
Commit: be88d574a9a49f384d4e80a661b1a7446ba6a92e
Parents: 62b42ef
Author: Allen Wittenauer 
Authored: Mon Nov 28 15:24:44 2016 -0800
Committer: Allen Wittenauer 
Committed: Mon Nov 28 15:24:44 2016 -0800

--
 .../hadoop-mapreduce-client-nativetask/pom.xml  |   10 +-
 .../src/CMakeLists.txt  |2 +
 .../src/main/native/gtest/gtest-all.cc  | 1483 ---
 .../src/main/native/gtest/include/gtest/gtest.h | 2341 +-
 4 files changed, 2892 insertions(+), 944 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
index 89b23cc..5f33843 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/pom.xml
@@ -207,12 +207,16 @@
 test
 cmake-test
 
-  test-native-task
+  
+  nativetask-nttest
   
${project.build.directory}/native/test
   ${basedir}/src
   
${project.build.directory}/native/test/nttest
-  --gtest_filter=-Perf.
-  
${project.build.directory}/native/test/native-results
+  
+--gtest_filter=-Perf.
+
--gtest_output=xml:${project.build.directory}/surefire-reports/TEST-nativetask-nttest.xml
+  
+  
${project.build.directory}/surefire-reports
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index f878a94..ee02062 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -98,7 +98,9 @@ include_directories(
 # add gtest as system library to suppress gcc warnings
 include_directories(SYSTEM ${SRC}/gtest/include)
 
+set(CMAKE_MACOSX_RPATH TRUE)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
 # macosx does not have -lrt

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be88d574/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
index a9a03b2..4f8c08a 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/gtest/gtest-all.cc
@@ -316,6 +316,8 @@ class GTEST_API_ SingleFailureChecker {
 #include 
 #include 
 #include 
+#include 
+#include 
 #include   // NOLINT
 #include 
 #include 
@@ -350,6 +352,7 @@ class GTEST_API_ SingleFailureChecker {
 #elif GTEST_OS_WINDOWS_MOBILE  // We are on Windows CE.
 
 # include   // NOLINT
+# undef min
 
 #elif GTEST_OS_WINDOWS  // We are on Windows proper.
 
@@ -372,6 +375,7 @@ class GTEST_API_ SingleFailureChecker {
 // cpplint thinks that the header is already included, so we want to
 // silence it.
 # include   // NOLINT
+# undef min
 
 #else
 
@@ -394,6 +398,8 @@ class GTEST_API_ SingleFailureChecker {
 #if 

hadoop git commit: YARN-5725. Test uncaught exception in TestContainersMonitorResourceChange.testContainersResourceChange when setting IP and host (Contributed by Miklos Szegedi via Daniel Templeton)

[templedf]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a2b1ff025 -> 62b42ef5d


YARN-5725. Test uncaught exception in 
TestContainersMonitorResourceChange.testContainersResourceChange
when setting IP and host (Contributed by Miklos Szegedi via Daniel Templeton)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62b42ef5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62b42ef5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62b42ef5

Branch: refs/heads/trunk
Commit: 62b42ef5dd04d516d33bf0890ac5cd49f8184a73
Parents: a2b1ff0
Author: Daniel Templeton 
Authored: Mon Nov 28 15:16:43 2016 -0800
Committer: Daniel Templeton 
Committed: Mon Nov 28 15:19:52 2016 -0800

--
 .../monitor/ContainersMonitorImpl.java  | 36 +---
 .../TestContainersMonitorResourceChange.java| 16 +++--
 2 files changed, 36 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/62b42ef5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index dda55d2..718604d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -536,15 +536,21 @@ public class ContainersMonitorImpl extends 
AbstractService implements
   }
 
   Container container = context.getContainers().get(containerId);
-  String[] ipAndHost = containerExecutor.getIpAndHost(container);
-  if (ipAndHost != null && ipAndHost[0] != null
-  && ipAndHost[1] != null) {
-container.setIpAndHost(ipAndHost);
-LOG.info(containerId + "'s ip = " + ipAndHost[0]
-+ ", and hostname = " + ipAndHost[1]);
+
+  if (container != null) {
+String[] ipAndHost = containerExecutor.getIpAndHost(container);
+
+if ((ipAndHost != null) && (ipAndHost[0] != null) &&
+(ipAndHost[1] != null)) {
+  container.setIpAndHost(ipAndHost);
+  LOG.info(containerId + "'s ip = " + ipAndHost[0]
+  + ", and hostname = " + ipAndHost[1]);
+} else {
+  LOG.info("Can not get both ip and hostname: "
+  + Arrays.toString(ipAndHost));
+}
   } else {
-LOG.info("Can not get both ip and hostname: " + Arrays
-.toString(ipAndHost));
+LOG.info(containerId + " is missing. Not setting ip and hostname");
   }
 }
   }
@@ -683,11 +689,15 @@ public class ContainersMonitorImpl extends 
AbstractService implements
 long currentPmemUsage, float cpuUsagePercentPerCore) {
   ContainerImpl container =
   (ContainerImpl) context.getContainers().get(containerId);
-  NMTimelinePublisher nmMetricsPublisher =
-  container.getNMTimelinePublisher();
-  if (nmMetricsPublisher != null) {
-nmMetricsPublisher.reportContainerResourceUsage(container,
-currentPmemUsage, cpuUsagePercentPerCore);
+  if (container != null) {
+NMTimelinePublisher nmMetricsPublisher =
+container.getNMTimelinePublisher();
+if (nmMetricsPublisher != null) {
+  nmMetricsPublisher.reportContainerResourceUsage(container,
+  currentPmemUsage, cpuUsagePercentPerCore);
+}
+  } else {
+LOG.info(containerId + " does not exist to report");
   }
 }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/62b42ef5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
--
diff --git 

[20/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
new file mode 100644
index 000..9767430
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.List;
+public interface ProviderCore {
+
+  String getName();
+
+  List getRoles();
+
+  Configuration getConf();
+
+  /**
+   * Verify that an instance definition is considered valid by the provider
+   * @param instanceDefinition instance definition
+   * @throws SliderException if the configuration is not valid
+   */
+  void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+  SliderException;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
new file mode 100644
index 000..761ac0f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.slider.api.ResourceKeys;
+
+/**
+ * Provider role and key for use in app requests.
+ * 
+ * This class uses the role name as the key for hashes and in equality tests,
+ * and ignores the other values.
+ */
+public final class ProviderRole {
+  public final String name;
+  public final String group;
+  public final int id;
+  public int placementPolicy;
+  public int nodeFailureThreshold;
+  public final long placementTimeoutSeconds;
+  public final String labelExpression;
+
+  public ProviderRole(String name, int id) {
+this(name,
+name,
+id,
+PlacementPolicy.DEFAULT,
+ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
+ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
+ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
+  }
+
+  /**
+   * Create a provider role
+   * @param name role/component name
+   

[59/63] [abbrv] hadoop git commit: YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by Billie Rinaldi

[jianhe]

YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a9aa8a1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a9aa8a1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a9aa8a1

Branch: refs/heads/yarn-native-services
Commit: 8a9aa8a199c9cea9b06d63d418fc96d96406a11d
Parents: 29b1f4e
Author: Jian He 
Authored: Sun Oct 16 17:01:09 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../org/apache/slider/client/SliderClient.java  | 72 
 .../slider/core/buildutils/InstanceBuilder.java |  4 +
 .../apache/slider/providers/ProviderUtils.java  | 62 --
 .../providers/docker/DockerClientProvider.java  |  4 +-
 .../providers/docker/DockerProviderService.java | 87 
 5 files changed, 164 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9aa8a1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 2840c4b..94e51e5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -178,6 +178,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
+import java.io.Console;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -918,57 +919,56 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   return;
 }
 
-BufferedReader br = null;
-try {
-  for (Entry cred : tree.credentials.entrySet()) {
-String provider = cred.getKey()
-.replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
-.replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
-List aliases = cred.getValue();
-if (aliases == null || aliases.isEmpty()) {
-  continue;
-}
-Configuration c = new Configuration(conf);
-c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
-CredentialProvider credentialProvider = 
CredentialProviderFactory.getProviders(c).get(0);
-Set existingAliases = new 
HashSet<>(credentialProvider.getAliases());
-for (String alias : aliases) {
-  if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
-log.info("Credentials for " + alias + " found in " + provider);
-  } else {
-if (br == null) {
-  br = new BufferedReader(new InputStreamReader(System.in));
-}
-char[] pass = readPassword(alias, br);
-credentialProvider.createCredentialEntry(alias, pass);
-credentialProvider.flush();
-Arrays.fill(pass, ' ');
+Console console = System.console();
+for (Entry cred : tree.credentials.entrySet()) {
+  String provider = cred.getKey()
+  .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
+  .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
+  List aliases = cred.getValue();
+  if (aliases == null || aliases.isEmpty()) {
+continue;
+  }
+  Configuration c = new Configuration(conf);
+  c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+  CredentialProvider credentialProvider = 
CredentialProviderFactory.getProviders(c).get(0);
+  Set existingAliases = new 
HashSet<>(credentialProvider.getAliases());
+  for (String alias : aliases) {
+if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
+  log.info("Credentials for " + alias + " found in " + provider);
+} else {
+  if (console == null) {
+throw new IOException("Unable to input password for " + alias +
+" because System.console() is null; provider " + provider +
+" must be populated manually");
   }
+  char[] pass = readPassword(alias, console);
+  

[36/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
new file mode 100644
index 000..373d64d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -0,0 +1,34473 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: SliderClusterMessages.proto
+
+package org.apache.slider.api.proto;
+
+public final class Messages {
+  private Messages() {}
+  public static void registerAllExtensions(
+  com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public interface RoleInstanceStateOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// required string name = 1;
+/**
+ * required string name = 1;
+ */
+boolean hasName();
+/**
+ * required string name = 1;
+ */
+java.lang.String getName();
+/**
+ * required string name = 1;
+ */
+com.google.protobuf.ByteString
+getNameBytes();
+
+// optional string role = 2;
+/**
+ * optional string role = 2;
+ */
+boolean hasRole();
+/**
+ * optional string role = 2;
+ */
+java.lang.String getRole();
+/**
+ * optional string role = 2;
+ */
+com.google.protobuf.ByteString
+getRoleBytes();
+
+// required uint32 state = 4;
+/**
+ * required uint32 state = 4;
+ */
+boolean hasState();
+/**
+ * required uint32 state = 4;
+ */
+int getState();
+
+// required uint32 exitCode = 5;
+/**
+ * required uint32 exitCode = 5;
+ */
+boolean hasExitCode();
+/**
+ * required uint32 exitCode = 5;
+ */
+int getExitCode();
+
+// optional string command = 6;
+/**
+ * optional string command = 6;
+ */
+boolean hasCommand();
+/**
+ * optional string command = 6;
+ */
+java.lang.String getCommand();
+/**
+ * optional string command = 6;
+ */
+com.google.protobuf.ByteString
+getCommandBytes();
+
+// optional string diagnostics = 7;
+/**
+ * optional string diagnostics = 7;
+ */
+boolean hasDiagnostics();
+/**
+ * optional string diagnostics = 7;
+ */
+java.lang.String getDiagnostics();
+/**
+ * optional string diagnostics = 7;
+ */
+com.google.protobuf.ByteString
+getDiagnosticsBytes();
+
+// repeated string output = 8;
+/**
+ * repeated string output = 8;
+ */
+java.util.List
+getOutputList();
+/**
+ * repeated string output = 8;
+ */
+int getOutputCount();
+/**
+ * repeated string output = 8;
+ */
+java.lang.String getOutput(int index);
+/**
+ * repeated string output = 8;
+ */
+com.google.protobuf.ByteString
+getOutputBytes(int index);
+
+// repeated string environment = 9;
+/**
+ * repeated string environment = 9;
+ */
+java.util.List
+getEnvironmentList();
+/**
+ * repeated string environment = 9;
+ */
+int getEnvironmentCount();
+/**
+ * repeated string environment = 9;
+ */
+java.lang.String getEnvironment(int index);
+/**
+ * repeated string environment = 9;
+ */
+com.google.protobuf.ByteString
+getEnvironmentBytes(int index);
+
+// required uint32 roleId = 10;
+/**
+ * required uint32 roleId = 10;
+ */
+boolean hasRoleId();
+/**
+ * required uint32 roleId = 10;
+ */
+int getRoleId();
+
+// required bool released = 11;
+/**
+ * required bool released = 11;
+ */
+boolean hasReleased();
+/**
+ * required bool released = 11;
+ */
+boolean getReleased();
+
+// required int64 createTime = 12;
+/**
+ * required int64 createTime = 12;
+ */
+boolean hasCreateTime();
+/**
+ * required int64 createTime = 12;
+ */
+long getCreateTime();
+
+// required int64 startTime = 13;
+/**
+ * required int64 startTime = 13;
+ */
+boolean hasStartTime();
+/**
+ * required int64 startTime = 13;
+ */
+long getStartTime();
+
+// required string host = 14;
+/**
+ * required string host = 14;
+ */
+boolean hasHost();
+/**
+ * required string host = 14;
+ */
+java.lang.String getHost();
+/**
+ * required string host = 14;
+ */
+

[55/63] [abbrv] hadoop git commit: YARN-5808. Add gc log options to the yarn daemon script when starting services-api. Contributed by Billie Rinaldi

[jianhe]

YARN-5808. Add gc log options to the yarn daemon script when starting 
services-api. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4945f85d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4945f85d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4945f85d

Branch: refs/heads/yarn-native-services
Commit: 4945f85dcf0a7502ab1bb750850bec76c458b458
Parents: 239c6a3
Author: Gour Saha 
Authored: Thu Nov 10 11:35:02 2016 -0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn | 16 
 hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 12 
 2 files changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4945f85d/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 26d54b8..2396a7a 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,7 @@ function hadoop_usage
   hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" "admin tools"
   hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
-  hadoop_add_subcommand "services-api" "run slider services api"
+  hadoop_add_subcommand "servicesapi" "run slider services api"
   hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager 
daemon"
   hadoop_add_subcommand "slider" "run a slider app"
   hadoop_add_subcommand "timelinereader" "run the timeline reader server"
@@ -144,20 +144,20 @@ function yarncmd_case
 scmadmin)
   HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
 ;;
-services-api)
+servicesapi)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 sharedcachemanager)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -166,15 +166,15 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
 slider)
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   HADOOP_CLASSNAME='org.apache.slider.Slider'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 timelinereader)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4945f85d/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh 
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index d003adb..3828897 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -136,3 +136,15 @@
 # See ResourceManager for some examples
 #
 #export YARN_SHAREDCACHEMANAGER_OPTS=
+
+###
+# Services API specific parameters
+###
+# Specify the JVM options to be used when starting the services API.
+#
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails 
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps 

[62/63] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40b660/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8203cf0..fdc5be1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -82,6 +82,8 @@ import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import static 
org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
+
 /** This class implements  the client-side aspects of the agent deployer */
 public class AgentClientProvider extends AbstractClientProvider
 implements AgentKeys, SliderKeys {
@@ -132,13 +134,13 @@ public class AgentClientProvider extends 
AbstractClientProvider
 sliderFileSystem.verifyFileExists(appDefPath);
 
 String agentConf = instanceDefinition.getAppConfOperations().
-getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
+getGlobalOptions().getOption(AGENT_CONF, "");
 if (StringUtils.isNotEmpty(agentConf)) {
   sliderFileSystem.verifyFileExists(new Path(agentConf));
 }
 
 String appHome = instanceDefinition.getAppConfOperations().
-getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+getGlobalOptions().get(PACKAGE_PATH);
 if (SliderUtils.isUnset(appHome)) {
   String agentImage = instanceDefinition.getInternalOperations().
   get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
@@ -173,7 +175,7 @@ public class AgentClientProvider extends 
AbstractClientProvider
 }
 
 Set names = resources.getComponentNames();
-names.remove(SliderKeys.COMPONENT_AM);
+names.remove(COMPONENT_AM);
 Map priorityMap = new HashMap();
 
 for (String name : names) {
@@ -271,7 +273,7 @@ public class AgentClientProvider extends 
AbstractClientProvider
 String agentImage = instanceDefinition.getInternalOperations().
 get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
 if (SliderUtils.isUnset(agentImage)) {
-  Path agentPath = new Path(tempPath.getParent(), 
AgentKeys.PROVIDER_AGENT);
+  Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
   log.info("Automatically uploading the agent tarball at {}", agentPath);
   fileSystem.getFileSystem().mkdirs(agentPath);
   if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
@@ -284,6 +286,12 @@ public class AgentClientProvider extends 
AbstractClientProvider
 
   @Override
   public Set getApplicationTags(SliderFileSystem fileSystem,
+  ConfTreeOperations appConf) throws SliderException {
+return getApplicationTags(fileSystem,
+getApplicationDefinitionPath(appConf));
+  }
+
+  public Set getApplicationTags(SliderFileSystem fileSystem,
 String appDef) throws SliderException {
 Set tags;
 Metainfo metaInfo = getMetainfo(fileSystem, appDef);
@@ -437,19 +445,19 @@ public class AgentClientProvider extends 
AbstractClientProvider
 if (config != null) {
   try {
 clientRoot = config.getJSONObject("global")
-.getString(AgentKeys.APP_CLIENT_ROOT);
+.getString(APP_CLIENT_ROOT);
   } catch (JSONException e) {
 log.info("Couldn't read {} from provided client config, falling " +
-"back on default", AgentKeys.APP_CLIENT_ROOT);
+"back on default", APP_CLIENT_ROOT);
   }
 }
 if (clientRoot == null && defaultConfig != null) {
   try {
 clientRoot = defaultConfig.getJSONObject("global")
-.getString(AgentKeys.APP_CLIENT_ROOT);
+.getString(APP_CLIENT_ROOT);
   } catch (JSONException e) {
 log.info("Couldn't read {} from default client config, using {}",
-AgentKeys.APP_CLIENT_ROOT, clientInstallPath);
+APP_CLIENT_ROOT, clientInstallPath);
   }
 }
 if (clientRoot == null) {
@@ -500,7 +508,7 @@ public class AgentClientProvider extends 
AbstractClientProvider
 try {
   String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + 

[58/63] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

YARN-5610. Initial code for native services REST API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6698c170
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6698c170
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6698c170

Branch: refs/heads/yarn-native-services
Commit: 6698c170357c29ee31800e0c89652c938a6276cb
Parents: c21882c
Author: Jian He 
Authored: Tue Oct 11 11:36:57 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 hadoop-project/pom.xml  |   20 +
 .../dev-support/findbugs-exclude.xml|   20 +
 .../hadoop-yarn-services-api/pom.xml|  225 +++
 .../yarn/services/api/ApplicationApi.java   |   38 +
 .../api/impl/ApplicationApiService.java | 1527 ++
 .../yarn/services/resource/Application.java |  452 ++
 .../services/resource/ApplicationState.java |   25 +
 .../services/resource/ApplicationStatus.java|  147 ++
 .../hadoop/yarn/services/resource/Artifact.java |  155 ++
 .../yarn/services/resource/BaseResource.java|   48 +
 .../yarn/services/resource/Component.java   |  377 +
 .../yarn/services/resource/ConfigFile.java  |  190 +++
 .../yarn/services/resource/Configuration.java   |  147 ++
 .../yarn/services/resource/Container.java   |  256 +++
 .../yarn/services/resource/ContainerState.java  |   25 +
 .../hadoop/yarn/services/resource/Error.java|  125 ++
 .../yarn/services/resource/PlacementPolicy.java |   97 ++
 .../yarn/services/resource/ReadinessCheck.java  |  161 ++
 .../hadoop/yarn/services/resource/Resource.java |  149 ++
 .../yarn/services/utils/RestApiConstants.java   |   66 +
 .../services/utils/RestApiErrorMessages.java|   79 +
 .../services/webapp/ApplicationApiWebApp.java   |  127 ++
 .../src/main/resources/log4j-server.properties  |   76 +
 .../resources/webapps/services-rest-api/app |   16 +
 .../src/main/scripts/run_rest_service.sh|   28 +
 .../src/main/webapp/WEB-INF/web.xml |   36 +
 .../api/impl/TestApplicationApiService.java |  232 +++
 .../hadoop-yarn-applications/pom.xml|2 +-
 28 files changed, 4845 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6698c170/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index fa43afd..5c46c6f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -126,6 +126,9 @@
 1.0-alpha-8
 900
 1.11.45
+
+1.5.4
+
1.4
   
 
   
@@ -1235,6 +1238,23 @@
   kerb-simplekdc
   1.0.0-RC2
 
+
+  
+io.swagger
+swagger-annotations
+${swagger-annotations-version}
+  
+  
+com.fasterxml.jackson.jaxrs
+jackson-jaxrs-json-provider
+${jackson2.version}
+  
+  
+org.apache.maven.doxia
+doxia-module-markdown
+${maven-doxia-module-markdown.version}
+  
+
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6698c170/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
new file mode 100644
index 000..b89146a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6698c170/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
new file mode 100644
index 000..78b7855
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -0,0 +1,225 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/maven-v4_0_0.xsd;>
+  4.0.0
+  
+org.apache.hadoop
+hadoop-yarn-applications
+3.0.0-alpha2-SNAPSHOT
+  
+  org.apache.hadoop
+  hadoop-yarn-services-api
+  Apache Hadoop YARN Services API
+  

[33/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
new file mode 100644
index 000..9879d05
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Serialized information about liveness
+ * 
+ *   If true liveness probes are implemented, this
+ *   datatype can be extended to publish their details.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ApplicationLivenessInformation {
+  /** flag set if the cluster is at size */
+  public boolean allRequestsSatisfied;
+
+  /** number of outstanding requests: those needed to satisfy */
+  public int requestsOutstanding;
+
+  /** number of requests submitted to YARN */
+  public int activeRequests;
+
+  @Override
+  public String toString() {
+final StringBuilder sb =
+new StringBuilder("ApplicationLivenessInformation{");
+sb.append("allRequestsSatisfied=").append(allRequestsSatisfied);
+sb.append(", requestsOutstanding=").append(requestsOutstanding);
+sb.append('}');
+return sb.toString();
+  }
+}
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
new file mode 100644
index 000..c46a59f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Serializable version of component data.
+ * 
+ * This is sent in REST 

[42/63] [abbrv] hadoop git commit: YARN-5796. Convert enums values in service code to upper case and special handling of an error. Contributed by Gour Saha

[jianhe]

YARN-5796. Convert enums values in service code to upper case and special 
handling of an error. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60ee3f55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60ee3f55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60ee3f55

Branch: refs/heads/yarn-native-services
Commit: 60ee3f55069f4681f9662e994dc7e08bd57f8e2a
Parents: d72e35a
Author: Jian He 
Authored: Tue Nov 1 11:00:11 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../yarn/services/api/impl/ApplicationApiService.java |  4 
 .../apache/hadoop/yarn/services/resource/Artifact.java|  2 +-
 .../apache/hadoop/yarn/services/resource/ConfigFile.java  |  4 ++--
 .../hadoop/yarn/services/resource/ReadinessCheck.java |  2 +-
 .../hadoop/yarn/services/utils/RestApiErrorMessages.java  | 10 +++---
 5 files changed, 15 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ee3f55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index cf43ac2..37bd134 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -172,6 +172,10 @@ public class ApplicationApiService implements 
ApplicationApi {
 applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE);
 return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
 .build();
+  } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS);
+return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+.build();
   } else {
 applicationStatus.setDiagnostics(se.getMessage());
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ee3f55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
index 9ac2bc7..87fcf89 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -40,7 +40,7 @@ public class Artifact implements Serializable {
   private String id = null;
 
   public enum TypeEnum {
-DOCKER("docker"), TARBALL("tarball"), APPLICATION("application");
+DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
 
 private String value;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/60ee3f55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
index 3ced153..01d976f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
+++ 

[60/63] [abbrv] hadoop git commit: Rename org.apache.slider.core.build to org.apache.slider.core.buildutils

[jianhe]

Rename org.apache.slider.core.build to org.apache.slider.core.buildutils


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12ee7a7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12ee7a7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12ee7a7c

Branch: refs/heads/yarn-native-services
Commit: 12ee7a7cf91954449d87da06b83b2cdafccdc29b
Parents: a379904
Author: Jian He 
Authored: Wed Aug 10 00:39:58 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../org/apache/slider/client/SliderClient.java  |   5 +-
 .../slider/core/buildutils/BuildHelper.java |  48 +++
 .../slider/core/buildutils/InstanceBuilder.java | 312 +++
 .../slider/core/buildutils/InstanceIO.java  |  83 +
 .../server/appmaster/SliderAppMaster.java   |   5 +-
 5 files changed, 446 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/12ee7a7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8210f4d..d464ce0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -115,8 +115,8 @@ import org.apache.slider.common.tools.Duration;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.common.tools.SliderVersionInfo;
-import org.apache.slider.core.build.InstanceBuilder;
-import org.apache.slider.core.build.InstanceIO;
+import org.apache.slider.core.buildutils.InstanceBuilder;
+import org.apache.slider.core.buildutils.InstanceIO;
 import org.apache.slider.core.conf.AggregateConf;
 import org.apache.slider.core.conf.ConfTree;
 import org.apache.slider.core.conf.ConfTreeOperations;
@@ -151,7 +151,6 @@ import org.apache.slider.core.registry.YarnAppListClient;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
 import org.apache.slider.core.registry.docstore.PublishedConfigSet;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
 import org.apache.slider.core.registry.docstore.PublishedExports;
 import org.apache.slider.core.registry.docstore.PublishedExportsOutputter;
 import org.apache.slider.core.registry.docstore.PublishedExportsSet;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/12ee7a7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
new file mode 100644
index 000..80f165f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/buildutils/BuildHelper.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 

[53/63] [abbrv] hadoop git commit: YARN-5513. Move Java only tests from slider develop to yarn-native-services. Contributed by Gour Saha

[jianhe]

YARN-5513. Move Java only tests from slider develop to yarn-native-services. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56715f7d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56715f7d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56715f7d

Branch: refs/heads/yarn-native-services
Commit: 56715f7dbd1c3661b22c46785bc6659ef0f6470c
Parents: 2f03062
Author: Jian He 
Authored: Wed Aug 17 00:42:24 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../dev-support/findbugs-exclude.xml|  20 +
 .../hadoop-yarn-slider-core/pom.xml |  22 +
 .../slider/common/tools/TestSliderUtils.java| 159 
 .../core/launch/TestAppMasterLauncher.java  | 157 
 .../TestAppMasterLauncherWithAmReset.java   |  92 ++
 .../TestPublishedConfigurationOutputter.java| 222 +
 .../agent/TestAgentClientProvider.java  |  77 ++
 .../agent/TestAgentLaunchParameter.java |  76 ++
 .../slider/providers/agent/TestAgentUtils.java  |  94 ++
 .../agent/TestAppDefinitionPersister.java   | 264 ++
 .../agent/TestComponentTagProvider.java | 115 +++
 .../slider/providers/agent/TestState.java   |  33 +
 .../application/metadata/TestConfigParser.java  | 107 +++
 .../metadata/TestMetainfoParser.java| 177 
 .../appmaster/TestServiceRecordAttributes.java  |  68 ++
 .../publisher/TestAgentProviderService.java |  60 ++
 .../publisher/TestSliderProviderFactory.java|  40 +
 .../server/servicemonitor/TestPortProbe.java|  37 +
 .../security/TestCertificateManager.java| 540 +++
 .../TestMultiThreadedStoreGeneration.java   | 156 
 .../server/services/workflow/MockService.java   |  80 ++
 .../workflow/ParentWorkflowTestBase.java|  70 ++
 .../workflow/ProcessCommandFactory.java |  96 ++
 .../services/workflow/SimpleRunnable.java   |  46 +
 .../workflow/TestWorkflowClosingService.java| 116 +++
 .../workflow/TestWorkflowCompositeService.java  | 113 +++
 .../workflow/TestWorkflowExecutorService.java   |  66 ++
 .../workflow/TestWorkflowRpcService.java| 107 +++
 .../workflow/TestWorkflowSequenceService.java   | 151 
 .../TestWorkflowServiceTerminatingRunnable.java |  64 ++
 .../workflow/WorkflowServiceTestBase.java   | 139 +++
 .../apache/slider/test/ContractTestUtils.java   | 901 +++
 .../slider/test/MiniZooKeeperCluster.java   | 395 
 .../org/apache/slider/tools/TestUtility.java| 181 
 .../slider/common/tools/test/metainfo.txt   |  16 +
 .../slider/common/tools/test/metainfo.xml   |  98 ++
 .../slider/common/tools/test/someOtherFile.txt  |  16 +
 .../slider/common/tools/test/someOtherFile.xml  |  17 +
 .../agent/application/metadata/metainfo.xml | 180 
 39 files changed, 5368 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/56715f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
new file mode 100644
index 000..b89146a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56715f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 591a5ca..d778f44 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -129,6 +129,13 @@
 
 
   org.apache.hadoop
+  hadoop-common
+  test-jar
+  test
+
+
+
+  org.apache.hadoop
   hadoop-hdfs
 
 
@@ -268,6 +275,20 @@
 
 
 
+  org.easymock
+  easymock
+  3.1
+  test
+
+
+
+  org.powermock
+  powermock-api-easymock
+  1.5
+  test
+
+
+

[37/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
new file mode 100644
index 000..f8e5e7c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -0,0 +1,795 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+
+/**
+ * Represents a cluster specification; designed to be sendable over the wire
+ * and persisted in JSON by way of Jackson.
+ * 
+ * When used in cluster status operations the info
+ * and statistics maps contain information about the cluster.
+ * 
+ * As a wire format it is less efficient in both xfer and ser/deser than 
+ * a binary format, but by having one unified format for wire and persistence,
+ * the code paths are simplified.
+ *
+ * This was the original single-file specification/model used in the Hoya
+ * precursor to Slider. Its now retained primarily as a way to publish
+ * the current state of the application, or at least a fraction thereof ...
+ * the larger set of information from the REST API is beyond the scope of
+ * this structure.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class ClusterDescription implements Cloneable {
+  protected static final Logger
+log = LoggerFactory.getLogger(ClusterDescription.class);
+
+  private static final String UTF_8 = "UTF-8";
+
+  /**
+   * version counter
+   */
+  public String version = "1.0";
+
+  /**
+   * Name of the cluster
+   */
+  public String name;
+
+  /**
+   * Type of cluster
+   */
+  public String type = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
+
+  /**
+   * State of the cluster
+   */
+  public int state;
+  
+  /*
+   State list for both clusters and nodes in them. Ordered so that destroyed 
follows
+   stopped.
+   
+   Some of the states are only used for recording
+   the persistent state of the cluster and are not
+   seen in node descriptions
+   */
+
+  /**
+   * Specification is incomplete & 

[48/63] [abbrv] hadoop git commit: YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha

[jianhe]

YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fac85edc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fac85edc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fac85edc

Branch: refs/heads/yarn-native-services
Commit: fac85edc3aa7b2df48cf294374e792b0fc1575c3
Parents: ace62eb
Author: Jian He 
Authored: Fri Oct 14 13:47:38 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../services/api/impl/ApplicationApiService.java| 16 ++--
 .../hadoop/yarn/services/resource/Application.java  |  6 +++---
 .../hadoop/yarn/services/resource/Artifact.java |  4 +++-
 .../hadoop/yarn/services/resource/Component.java|  4 +++-
 .../hadoop/yarn/services/resource/ConfigFile.java   |  4 +++-
 .../yarn/services/resource/Configuration.java   |  4 +++-
 .../hadoop/yarn/services/resource/Container.java|  6 +++---
 .../yarn/services/resource/PlacementPolicy.java |  4 +++-
 .../yarn/services/resource/ReadinessCheck.java  |  4 +++-
 .../hadoop/yarn/services/resource/Resource.java |  2 +-
 10 files changed, 39 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac85edc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 0a62629..21cf113 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -276,7 +276,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 
 // If it is a simple app with no components, then create a default 
component
 if (application.getComponents() == null) {
-  application.setComponents(getDefaultComponentAsList());
+  application.setComponents(getDefaultComponentAsList(application));
 }
 
 // Application lifetime if not specified, is set to unlimited lifetime
@@ -1029,7 +1029,8 @@ public class ApplicationApiService implements 
ApplicationApi {
 // end-users point of view, is out of scope of the REST API. Also, this
 // readiness has nothing to do with readiness-check defined at the 
component
 // level (which is used for dependency resolution of component DAG).
-if (totalNumberOfIpAssignedContainers == 
totalExpectedNumberOfRunningContainers) {
+if (totalNumberOfIpAssignedContainers
+.longValue() == totalExpectedNumberOfRunningContainers.longValue()) {
   app.setState(ApplicationState.READY);
 }
 logger.info("Application = {}", app);
@@ -1389,6 +1390,17 @@ public class ApplicationApiService implements 
ApplicationApi {
 return Response.status(Status.NO_CONTENT).build();
   }
 
+  // create default component and initialize with app level global values
+  private List getDefaultComponentAsList(Application app) {
+List comps = getDefaultComponentAsList();
+Component comp = comps.get(0);
+comp.setArtifact(app.getArtifact());
+comp.setResource(app.getResource());
+comp.setNumberOfContainers(app.getNumberOfContainers());
+comp.setLaunchCommand(app.getLaunchCommand());
+return comps;
+  }
+
   private List getDefaultComponentAsList() {
 Component comp = new Component();
 comp.setName(DEFAULT_COMPONENT_NAME);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fac85edc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index 719bf95..ed65ad2 100644
--- 

[28/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
new file mode 100644
index 000..5140059
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+  /**
+   * This is a listing of the roles to create
+   */
+  @Parameter(names = {ARG_COMPONENT,  ARG_COMPONENT_SHORT, ARG_ROLE},
+ arity = 2,
+ description = "--component   e.g. +1 incr by 1, -2 
decr by 2, and 3 makes final count 3",
+ splitter = DontSplitArguments.class)
+  public List componentTuples = new ArrayList<>(0);
+
+
+  /**
+   * Get the role mapping (may be empty, but never null)
+   * @return role mapping
+   * @throws BadCommandArgumentsException parse problem
+   */
+  public Map getComponentMap() throws 
BadCommandArgumentsException {
+return convertTupleListToMap("component", componentTuples);
+  }
+
+  public List getComponentTuples() {
+return componentTuples;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
new file mode 100644
index 000..0344305
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+  @Override
+  public List split(String value) {
+List list = new ArrayList<>(1);
+list.add(value);
+return list;
+  }
+}

[35/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
new file mode 100644
index 000..17fd965
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.proto;
+
+import com.google.protobuf.ByteString;
+import org.apache.commons.io.IOUtils;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeEntryInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.persist.AggregateConfSerDeser;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.server.services.security.SecurityStore;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to handle marshalling of REST
+ * types to/from Protobuf records.
+ */
+public class RestTypeMarshalling {
+
+  public static Messages.ApplicationLivenessInformationProto
+  marshall(ApplicationLivenessInformation info) {
+
+Messages.ApplicationLivenessInformationProto.Builder builder =
+Messages.ApplicationLivenessInformationProto.newBuilder();
+builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
+builder.setRequestsOutstanding(info.requestsOutstanding);
+return builder.build();
+  }
+
+  public static ApplicationLivenessInformation
+  unmarshall(Messages.ApplicationLivenessInformationProto wire) {
+ApplicationLivenessInformation info = new ApplicationLivenessInformation();
+info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
+info.requestsOutstanding = wire.getRequestsOutstanding();
+return info;
+  }
+
+  public static ComponentInformation
+  unmarshall(Messages.ComponentInformationProto wire) {
+ComponentInformation info = new ComponentInformation();
+info.name = wire.getName();
+info.priority = wire.getPriority();
+info.placementPolicy = wire.getPlacementPolicy();
+
+info.actual = wire.getActual();
+info.completed = wire.getCompleted();
+info.desired = wire.getDesired();
+info.failed = wire.getFailed();
+info.releasing = wire.getReleasing();
+info.requested = wire.getRequested();
+info.started = wire.getStarted();
+info.startFailed = wire.getStartFailed();
+info.totalRequested = wire.getTotalRequested();
+info.containers = new ArrayList<>(wire.getContainersList());
+if (wire.hasFailureMessage()) {
+  info.failureMessage = wire.getFailureMessage();
+}
+if (wire.hasPendingAntiAffineRequestCount()) {
+  info.pendingAntiAffineRequestCount = 
wire.getPendingAntiAffineRequestCount();
+}
+if (wire.hasIsAARequestOutstanding()) {
+  info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+}
+return info;
+  }
+
+  public static Messages.GetCertificateStoreResponseProto marshall(
+  SecurityStore securityStore) throws IOException {
+Messages.GetCertificateStoreResponseProto.Builder builder =
+

[50/63] [abbrv] hadoop git commit: YARN-5538. Apply SLIDER-875 to yarn-native-services. Contributed by Billie Rinaldi

[jianhe]

YARN-5538. Apply SLIDER-875 to yarn-native-services. Contributed by Billie 
Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bbb4b56
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bbb4b56
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bbb4b56

Branch: refs/heads/yarn-native-services
Commit: 6bbb4b5631f48746ef1d6e3727342735c2693a35
Parents: 56715f7
Author: Jian He 
Authored: Mon Aug 22 13:58:50 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../java/org/apache/slider/api/RoleKeys.java|   5 +
 .../org/apache/slider/client/SliderClient.java  |  14 +-
 .../org/apache/slider/common/SliderKeys.java|  13 ++
 .../apache/slider/common/tools/SliderUtils.java |  88 +++
 .../slider/core/buildutils/InstanceBuilder.java | 205 +
 .../slider/core/conf/ConfTreeOperations.java|  50 
 .../providers/AbstractProviderService.java  |   3 +-
 .../slider/providers/ProviderService.java   |   4 +-
 .../providers/agent/AgentClientProvider.java|  29 ++-
 .../slider/providers/agent/AgentKeys.java   |   3 +
 .../providers/agent/AgentProviderService.java   | 227 ++-
 .../slider/providers/agent/AgentUtils.java  |  16 ++
 .../providers/agent/ComponentCommandOrder.java  | 112 ++---
 .../server/appmaster/SliderAppMaster.java   |   6 +-
 .../slider/server/appmaster/state/AppState.java |   5 +
 .../appmaster/web/rest/agent/AgentResource.java |   4 +-
 16 files changed, 669 insertions(+), 115 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bbb4b56/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 812a6b3..ce413ff 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -35,6 +35,11 @@ public interface RoleKeys {
   String ROLE_GROUP = "role.group";
 
   /**
+   * The prefix of a role: {@value}
+   */
+  String ROLE_PREFIX = "role.prefix";
+
+  /**
* Status report: number actually granted : {@value} 
*/
   String ROLE_ACTUAL_INSTANCES = "role.actual.instances";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bbb4b56/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d464ce0..3129f6f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -151,6 +151,7 @@ import org.apache.slider.core.registry.YarnAppListClient;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
 import org.apache.slider.core.registry.docstore.PublishedConfigSet;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
 import org.apache.slider.core.registry.docstore.PublishedExports;
 import org.apache.slider.core.registry.docstore.PublishedExportsOutputter;
 import org.apache.slider.core.registry.docstore.PublishedExportsSet;
@@ -724,7 +725,8 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
 clustername, clusterDirectory);
 try {
-  checkForCredentials(getConfig(), instanceDefinition.getAppConf());
+  checkForCredentials(getConfig(), instanceDefinition.getAppConf(),
+  clustername);
 

[27/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
new file mode 100644
index 000..aa5edf1
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -0,0 +1,915 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.apache.slider.core.persist.Filenames;
+import org.apache.slider.core.persist.InstancePaths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import static 
org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
+import static 
org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
+
+public class CoreFileSystem {
+  private static final Logger
+log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+  private static final String UTF_8 = "UTF-8";
+
+  protected final FileSystem fileSystem;
+  protected final Configuration configuration;
+
+  public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+Preconditions.checkNotNull(fileSystem,
+   "Cannot create a CoreFileSystem with a null 
FileSystem");
+Preconditions.checkNotNull(configuration,
+   "Cannot create a CoreFileSystem with a null 
Configuration");
+this.fileSystem = fileSystem;
+this.configuration = configuration;
+  }
+
+  public CoreFileSystem(Configuration configuration) throws IOException {
+Preconditions.checkNotNull(configuration,
+   "Cannot create a CoreFileSystem with a null 
Configuration");
+this.fileSystem = FileSystem.get(configuration);
+this.configuration = fileSystem.getConf();
+  }
+  
+  /**
+   * Get the temp path for this cluster
+   * @param clustername name of the cluster
+   * @return path for temp files (is not purged)
+   */
+  public Path getTempPathForCluster(String clustername) {
+Path clusterDir = buildClusterDirPath(clustername);
+return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
+  }
+
+  /**
+   * Returns the 

[54/63] [abbrv] hadoop git commit: YARN-5775. Convert enums in swagger definition to uppercase. Contributed by Gour Saha

[jianhe]

YARN-5775. Convert enums in swagger definition to uppercase. Contributed by 
Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c2a7829
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c2a7829
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c2a7829

Branch: refs/heads/yarn-native-services
Commit: 5c2a7829031f7b34b459c9d91c7d6ef49baa5065
Parents: b036fb0
Author: Billie Rinaldi 
Authored: Tue Oct 25 11:25:51 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 38 ++--
 1 file changed, 19 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c2a7829/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 6169fcd..7eb3196 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -241,10 +241,10 @@ definitions:
 type: string
 description: Artifact type, like docker, tarball, etc. (optional).
 enum:
-  - docker
-  - tarball
-  - application
-default: docker
+  - DOCKER
+  - TARBALL
+  - APPLICATION
+default: DOCKER
   uri:
 type: string
 description: Artifact location to support multiple artifact stores 
(optional).
@@ -303,7 +303,7 @@ definitions:
 type: string
 description: E.g. HTTP (YARN will perform a simple REST call at a 
regular interval and expect a 204 No content).
 enum:
-  - http
+  - HTTP
   uri:
 type: string
 description: Fully qualified REST uri endpoint.
@@ -335,13 +335,13 @@ definitions:
 type: string
 description: Config file in the standard format like xml, properties, 
json, yaml, template.
 enum:
-  - xml
-  - properties
-  - json
-  - yaml
-  - template
-  - env
-  - hadoop_xml
+  - XML
+  - PROPERTIES
+  - JSON
+  - YAML
+  - TEMPLATE
+  - ENV
+  - HADOOP_XML
   dest_file:
 type: string
 description: The absolute path that this configuration file should be 
mounted as, in the application container.
@@ -386,11 +386,11 @@ definitions:
 type: string
 description: enum of the state of the application
 enum:
-  - accepted
-  - started
-  - ready
-  - stopped
-  - failed
+  - ACCEPTED
+  - STARTED
+  - READY
+  - STOPPED
+  - FAILED
   ContainerState:
 description: The current state of the container of an application.
 properties:
@@ -398,8 +398,8 @@ definitions:
 type: string
 description: enum of the state of the container
 enum:
-  - init
-  - ready
+  - INIT
+  - READY
   ApplicationStatus:
 description: The current status of a submitted application, returned as a 
response to the GET API.
 properties:


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[57/63] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6698c170/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
new file mode 100644
index 000..cfcae95
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.slider.providers.PlacementPolicy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+/**
+ * An Application resource has the following attributes.
+ **/
+
+@ApiModel(description = "An Application resource has the following 
attributes.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, 
containers " })
+public class Application extends BaseResource {
+  private static final long serialVersionUID = -4491694636566094885L;
+
+  private String id = null;
+  private String name = null;
+  private Artifact artifact = null;
+  private Resource resource = null;
+  private String launchCommand = null;
+  private Date launchTime = null;
+  private Long numberOfContainers = null;
+  private Long numberOfRunningContainers = null;
+  private Long lifetime = null;
+  private PlacementPolicy placementPolicy = null;
+  private List components = null;
+  private Configuration configuration = null;
+  private List containers = new ArrayList<>();
+  private ApplicationState state = null;
+  private Map quicklinks = null;
+  private String queue;
+
+  /**
+   * A unique application id.
+   **/
+  public Application id(String id) {
+this.id = id;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", required = true, value = "A unique 
application id.")
+  @JsonProperty("id")
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  /**
+   * A unique application name.
+   **/
+  public Application name(String name) {
+this.name = name;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", required = true, value = "A unique 
application name.")
+  @JsonProperty("name")
+  public String getName() {
+return name;
+  }
+
+  public void setName(String name) {
+this.name = name;
+  }
+
+  /**
+   * Artifact of single-component applications. Mandatory if components
+   * attribute is not specified.
+   **/
+  public Application artifact(Artifact artifact) {
+this.artifact = artifact;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Artifact of single-component 
applications. Mandatory if components attribute is not specified.")
+  @JsonProperty("artifact")
+  public Artifact getArtifact() {
+return artifact;
+  }
+
+  public void setArtifact(Artifact artifact) {
+this.artifact = artifact;
+  }
+
+  /**
+   * Resource of single-component applications or the global default for
+   * multi-component applications. Mandatory if it is a single-component
+   * application and if cpus and memory are not specified at the 

[41/63] [abbrv] hadoop git commit: YARN-5778. Add .keep file for yarn native services AM web app. Contributed by Billie Rinaldi

[jianhe]

YARN-5778. Add .keep file for yarn native services AM web app. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b036fb02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b036fb02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b036fb02

Branch: refs/heads/yarn-native-services
Commit: b036fb02f77a3cae77d0292a869bcf1b4e5bfe0c
Parents: 8a9aa8a
Author: Gour Saha 
Authored: Tue Oct 25 10:06:16 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../src/main/resources/webapps/slideram/.keep| 0
 1 file changed, 0 insertions(+), 0 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b036fb02/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
new file mode 100644
index 000..e69de29


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[38/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

YARN-5461. Initial code ported from slider-core module. (jianhe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a379904d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a379904d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a379904d

Branch: refs/heads/yarn-native-services
Commit: a379904d28836cd9247b2242cb1e287f8b789f8a
Parents: 5d5614f
Author: Jian He 
Authored: Wed Aug 3 00:18:01 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../hadoop-yarn-slider-core/pom.xml |   397 +
 .../src/assembly/executable-jar.xml |47 +
 .../src/license/THIRD-PARTY.properties  |33 +
 .../slider/server/avro/RoleHistoryRecord.avsc   |   114 +
 .../apache/hadoop/security/KerberosDiags.java   |   676 +
 .../src/main/java/org/apache/slider/Slider.java |52 +
 .../apache/slider/api/ClusterDescription.java   |   795 +
 .../slider/api/ClusterDescriptionKeys.java  |25 +
 .../api/ClusterDescriptionOperations.java   |93 +
 .../java/org/apache/slider/api/ClusterNode.java |   220 +
 .../org/apache/slider/api/InternalKeys.java |   199 +
 .../java/org/apache/slider/api/OptionKeys.java  |58 +
 .../org/apache/slider/api/ResourceKeys.java |   201 +
 .../java/org/apache/slider/api/RoleKeys.java|   116 +
 .../apache/slider/api/SliderApplicationApi.java |   167 +
 .../slider/api/SliderClusterProtocol.java   |   179 +
 .../java/org/apache/slider/api/StateValues.java |53 +
 .../java/org/apache/slider/api/StatusKeys.java  |   117 +
 .../org/apache/slider/api/proto/Messages.java   | 34473 +
 .../slider/api/proto/RestTypeMarshalling.java   |   319 +
 .../slider/api/proto/SliderClusterAPI.java  |  2293 ++
 .../types/ApplicationLivenessInformation.java   |53 +
 .../slider/api/types/ComponentInformation.java  |   107 +
 .../slider/api/types/ContainerInformation.java  |58 +
 .../slider/api/types/NodeEntryInformation.java  |78 +
 .../slider/api/types/NodeInformation.java   |59 +
 .../slider/api/types/NodeInformationList.java   |41 +
 .../slider/api/types/PingInformation.java   |47 +
 .../apache/slider/api/types/RoleStatistics.java |66 +
 .../api/types/SliderInstanceDescription.java|54 +
 .../slider/client/ClientRegistryBinder.java |   201 +
 .../org/apache/slider/client/ClientUtils.java   |   111 +
 .../org/apache/slider/client/SliderClient.java  |  4569 +++
 .../apache/slider/client/SliderClientAPI.java   |   368 +
 .../slider/client/SliderYarnClientImpl.java |   410 +
 .../apache/slider/client/TokensOperation.java   |   109 +
 .../client/ipc/SliderApplicationIpcClient.java  |   245 +
 .../client/ipc/SliderClusterOperations.java |   529 +
 .../slider/client/rest/BaseRestClient.java  |   152 +
 .../slider/client/rest/RestClientFactory.java   |89 +
 .../rest/SliderApplicationApiRestClient.java|   326 +
 .../org/apache/slider/common/Constants.java |35 +
 .../apache/slider/common/SliderExitCodes.java   |88 +
 .../org/apache/slider/common/SliderKeys.java|   278 +
 .../common/SliderXMLConfKeysForTesting.java |83 +
 .../apache/slider/common/SliderXmlConfKeys.java |   206 +
 .../common/params/AbstractActionArgs.java   |   178 +
 .../common/params/AbstractArgsDelegate.java |26 +
 .../AbstractClusterBuildingActionArgs.java  |   217 +
 .../common/params/ActionAMSuicideArgs.java  |44 +
 .../slider/common/params/ActionBuildArgs.java   |32 +
 .../slider/common/params/ActionClientArgs.java  |98 +
 .../slider/common/params/ActionCreateArgs.java  |59 +
 .../common/params/ActionDependencyArgs.java |65 +
 .../slider/common/params/ActionDestroyArgs.java |37 +
 .../common/params/ActionDiagnosticArgs.java |73 +
 .../slider/common/params/ActionEchoArgs.java|33 +
 .../slider/common/params/ActionExistsArgs.java  |47 +
 .../slider/common/params/ActionFlexArgs.java|54 +
 .../slider/common/params/ActionFreezeArgs.java  |56 +
 .../slider/common/params/ActionHelpArgs.java|51 +
 .../common/params/ActionInstallKeytabArgs.java  |57 +
 .../common/params/ActionInstallPackageArgs.java |58 +
 .../slider/common/params/ActionKDiagArgs.java   |86 +
 .../slider/common/params/ActionKeytabArgs.java  |81 +
 .../common/params/ActionKillContainerArgs.java  |37 +
 .../slider/common/params/ActionListArgs.java|74 +
 .../slider/common/params/ActionLookupArgs.java  |76 +
 .../slider/common/params/ActionNodesArgs.java   |71 +
 .../slider/common/params/ActionPackageArgs.java |81 +
 .../common/params/ActionRegistryArgs.java   |   218 +
 .../slider/common/params/ActionResolveArgs.java |   153 +
 .../common/params/ActionResourceArgs.java   |68 +
 

[49/63] [abbrv] hadoop git commit: YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi

[jianhe]

YARN-5690. Integrate native services modules into maven build. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d72e35a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d72e35a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d72e35a9

Branch: refs/heads/yarn-native-services
Commit: d72e35a91893d471e91adff0965987b137be5661
Parents: 51faaca
Author: Gour Saha 
Authored: Thu Oct 27 08:50:36 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../resources/assemblies/hadoop-yarn-dist.xml   | 26 
 .../assemblies/hadoop-yarn-services-api.xml | 36 +++
 .../assemblies/hadoop-yarn-slider-dist.xml  | 30 +
 hadoop-project/pom.xml  |  6 ++
 hadoop-yarn-project/hadoop-yarn/bin/yarn| 30 +
 .../hadoop-yarn-services-api/pom.xml| 44 +++--
 .../conf/slideram-log4j.properties  | 68 
 .../hadoop-yarn-slider-core/pom.xml | 38 +++
 .../org/apache/slider/client/SliderClient.java  | 29 +++--
 .../org/apache/slider/common/SliderKeys.java|  2 +-
 .../apache/slider/common/tools/SliderUtils.java | 48 ++
 .../providers/agent/AgentClientProvider.java|  3 +-
 .../slideram/SliderAMClientProvider.java| 15 +++--
 .../TestPublishedConfigurationOutputter.java| 10 ++-
 14 files changed, 343 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d72e35a9/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index c3f459c..552087c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -86,6 +86,32 @@
   
 
 
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target
+  
/share/hadoop/${hadoop.component}/sources
+  
+*-sources.jar
+  
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf
+  etc/hadoop
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}
+  
/share/hadoop/${hadoop.component}/lib/slider
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
+  
/share/hadoop/${hadoop.component}/sources
+  
+*-sources.jar
+  
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}
+  
/share/hadoop/${hadoop.component}/lib/services-api
+
+
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d72e35a9/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
new file mode 100644
index 000..589f724
--- /dev/null
+++ 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
@@ -0,0 +1,36 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0;
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+  
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
 http://maven.apache.org/xsd/assembly-1.1.0.xsd;>
+  hadoop-yarn-services-api-dist
+  
+dir
+  
+  false
+  
+
+  false
+  
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base
+
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
+
com.fasterxml.jackson.module:jackson-module-jaxb-annotations
+io.swagger:swagger-annotations
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d72e35a9/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
new file mode 100644
index 000..5de45a9
--- /dev/null
+++ 

[63/63] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi

[jianhe]

YARN-5505. Create an agent-less docker provider in the native-services 
framework. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb40b660
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb40b660
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb40b660

Branch: refs/heads/yarn-native-services
Commit: bb40b6609f4b84cda25a9459c8213e116ba42392
Parents: 6bbb4b5
Author: Jian He 
Authored: Thu Sep 1 22:38:42 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../java/org/apache/slider/api/OptionKeys.java  |   15 +-
 .../org/apache/slider/client/SliderClient.java  |   17 +-
 .../org/apache/slider/common/SliderKeys.java|   22 +-
 .../apache/slider/common/tools/SliderUtils.java |4 +
 .../slider/core/launch/AbstractLauncher.java|   18 +-
 .../PublishedConfigurationOutputter.java|6 +-
 .../providers/AbstractClientProvider.java   |4 +-
 .../providers/AbstractProviderService.java  |   22 +-
 .../slider/providers/ProviderService.java   |   12 +-
 .../apache/slider/providers/ProviderUtils.java  | 1391 ++
 .../providers/agent/AgentClientProvider.java|   36 +-
 .../slider/providers/agent/AgentKeys.java   |   12 +-
 .../providers/agent/AgentProviderService.java   |  705 ++---
 .../providers/docker/DockerClientProvider.java  |   96 ++
 .../slider/providers/docker/DockerKeys.java |   32 +
 .../providers/docker/DockerProviderFactory.java |   43 +
 .../providers/docker/DockerProviderService.java |  355 +
 .../slideram/SliderAMProviderService.java   |4 -
 .../server/appmaster/SliderAppMaster.java   |   39 +-
 .../main/resources/org/apache/slider/slider.xml |4 +
 .../slider/providers/docker/appConfig.json  |   42 +
 .../slider/providers/docker/resources.json  |   16 +
 .../slider/providers/docker/test.template   |   16 +
 23 files changed, 1971 insertions(+), 940 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40b660/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index a035a99..434b1d9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -41,7 +41,20 @@ public interface OptionKeys extends InternalKeys {
* Prefix for site.xml options: {@value}
*/
   String SITE_XML_PREFIX = "site.";
-
+  /**
+   * Prefix for config file options: {@value}
+   */
+  String CONF_FILE_PREFIX = "conf.";
+  /**
+   * Prefix for package options: {@value}
+   */
+  String PKG_FILE_PREFIX = "pkg.";
+  /**
+   * Prefix for export options: {@value}
+   */
+  String EXPORT_PREFIX = "export.";
+  String TYPE_SUFFIX = ".type";
+  String NAME_SUFFIX = ".name";
 
   /**
* Zookeeper quorum host list: {@value}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40b660/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3129f6f..5096bb7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -151,7 +151,6 @@ import org.apache.slider.core.registry.YarnAppListClient;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
 import org.apache.slider.core.registry.docstore.PublishedConfigSet;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;

[61/63] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb40b660/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
new file mode 100644
index 000..bebb5f0
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.docker;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.ConfigUtils;
+import org.apache.slider.core.registry.docstore.ExportEntry;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Scanner;
+
+public class DockerProviderService extends AbstractProviderService implements
+ProviderCore,
+DockerKeys,
+SliderKeys {
+
+  protected static final Logger log =
+  LoggerFactory.getLogger(DockerProviderService.class);
+  private static final ProviderUtils providerUtils = new ProviderUtils(log);
+  private static final String EXPORT_GROUP = "quicklinks";
+  private static final String APPLICATION_TAG = "application";
+
+  private String clusterName = null;
+  private SliderFileSystem fileSystem = null;
+
+  protected DockerProviderService() {
+super("DockerProviderService");
+  }
+
+  @Override
+  public List getRoles() {
+return Collections.emptyList();
+  }
+
+  @Override
+  public boolean isSupportedRole(String role) {
+return true;
+  }
+
+  @Override
+  public void validateInstanceDefinition(AggregateConf instanceDefinition)
+  throws SliderException {
+  }
+
+  private String getClusterName() {
+if (SliderUtils.isUnset(clusterName)) {
+  clusterName = 
getAmState().getInternalsSnapshot().get(OptionKeys.APPLICATION_NAME);
+}
+return clusterName;
+  }
+
+  @Override
+  public void 

[51/63] [abbrv] hadoop git commit: YARN-5513. Move Java only tests from slider develop to yarn-native-services. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56715f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
new file mode 100644
index 000..7eaaefe
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/test/ContractTestUtils.java
@@ -0,0 +1,901 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.test;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Assert;
+import org.junit.internal.AssumptionViolatedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.UUID;
+
+/**
+ * Utilities used across test cases to make assertions about filesystems
+ * -assertions which fail with useful information.
+ * This is lifted from Hadoop common Test; that JAR isn't published, so
+ * we have to make do.
+ */
+public class ContractTestUtils extends Assert {
+
+  private static final Logger LOG =
+  LoggerFactory.getLogger(ContractTestUtils.class);
+
+  public static final String IO_FILE_BUFFER_SIZE = "io.file.buffer.size";
+
+  // For scale testing, we can repeatedly write small chunk data to generate
+  // a large file.
+  public static final String IO_CHUNK_BUFFER_SIZE = "io.chunk.buffer.size";
+  public static final int DEFAULT_IO_CHUNK_BUFFER_SIZE = 128;
+  public static final String IO_CHUNK_MODULUS_SIZE = "io.chunk.modulus.size";
+  public static final int DEFAULT_IO_CHUNK_MODULUS_SIZE = 128;
+
+  /**
+   * Assert that a property in the property set matches the expected value
+   * @param props property set
+   * @param key property name
+   * @param expected expected value. If null, the property must not be in the 
set
+   */
+  public static void assertPropertyEquals(Properties props,
+  String key,
+  String expected) {
+String val = props.getProperty(key);
+if (expected == null) {
+  assertNull("Non null property " + key + " = " + val, val);
+} else {
+  assertEquals("property " + key + " = " + val,
+  expected,
+  val);
+}
+  }
+
+  /**
+   *
+   * Write a file and read it in, validating the result. Optional flags control
+   * whether file overwrite operations should be enabled, and whether the
+   * file should be deleted afterwards.
+   *
+   * If there is a mismatch between what was written and what was expected,
+   * a small range of bytes either side of the first error are logged to aid
+   * diagnosing what problem occurred -whether it was a previous file
+   * or a corrupting of the current file. This assumes that two
+   * sequential runs to the same path use datasets with different character
+   * moduli.
+   *
+   * @param fs filesystem
+   * @param path path to write to
+   * @param len length of data
+   * @param overwrite should the create option allow overwrites?
+   * @param delete should the file be deleted afterwards? -with a verification
+   * that it worked. Deletion is not attempted if an assertion has failed
+   * earlier -it is not in a finally{} block.
+   * @throws IOException IO problems
+   */
+  public static void writeAndRead(FileSystem fs,
+  Path path,
+   

[31/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
new file mode 100644
index 000..30f6ba9
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.types.NodeInformationList;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionClientArgs;
+import org.apache.slider.common.params.ActionDependencyArgs;
+import org.apache.slider.common.params.ActionDestroyArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionEchoArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionInstallKeytabArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
+import org.apache.slider.common.params.ActionKeytabArgs;
+import org.apache.slider.common.params.ActionNodesArgs;
+import org.apache.slider.common.params.ActionPackageArgs;
+import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionResourceArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionUpgradeArgs;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface of those method calls in the slider API that are intended
+ * for direct public invocation.
+ * 
+ * Stability: evolving
+ */
+public interface SliderClientAPI extends Service {
+  /**
+   * Destroy a cluster. There's two race conditions here
+   * #1 the cluster is started between verifying that there are no live
+   * clusters of that name.
+   */
+  int actionDestroy(String clustername, ActionDestroyArgs destroyArgs)
+  throws YarnException, IOException;
+
+  int actionDestroy(String clustername) throws YarnException,
+  IOException;
+
+  /**
+   * AM to commit an asynchronous suicide
+   */
+  int actionAmSuicide(String clustername,
+  ActionAMSuicideArgs args) throws YarnException, IOException;
+
+  /**
+   * Get the provider for this cluster
+   * @param provider the name of the provider
+   * @return the provider instance
+   * @throws SliderException problems building the provider
+   */
+  AbstractClientProvider createClientProvider(String provider)
+throws SliderException;
+
+  /**
+   * Build up the cluster specification/directory
+   *
+   * @param clustername cluster name
+   * @param buildInfo the arguments needed to build the cluster
+   * @throws YarnException Yarn problems
+   * @throws IOException other 

[45/63] [abbrv] hadoop git commit: YARN-5828. Native services client errors out when config formats are uppercase. Contributed by Billie Rinaldi

[jianhe]

YARN-5828. Native services client errors out when config formats are uppercase. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/239c6a3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/239c6a3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/239c6a3e

Branch: refs/heads/yarn-native-services
Commit: 239c6a3e4c103db3d7072e5756aa2c2d1ee4e24f
Parents: ea69a4b
Author: Gour Saha 
Authored: Thu Nov 3 18:15:44 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../org/apache/slider/core/registry/docstore/ConfigFormat.java   | 4 +++-
 .../src/main/java/org/apache/slider/providers/ProviderUtils.java | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/239c6a3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index ddab606..723b975 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -18,6 +18,8 @@
 
 package org.apache.slider.core.registry.docstore;
 
+import java.util.Locale;
+
 public enum ConfigFormat {
 
   JSON("json"),
@@ -51,7 +53,7 @@ public enum ConfigFormat {
*/
   public static ConfigFormat resolve(String type) {
 for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type)) {
+  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
 return format;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/239c6a3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index c5e6782..39986c1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -611,7 +611,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
   }
   ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
   if (configFormat == null) {
-throw new BadConfigException("Config format " + configFormat +
+throw new BadConfigException("Config format " + configFileType +
 " doesn't exist");
   }
   localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[44/63] [abbrv] hadoop git commit: YARN-5623. Apply SLIDER-1166 to yarn-native-services branch. Contributed by Gour Saha

[jianhe]

YARN-5623. Apply SLIDER-1166 to yarn-native-services branch. Contributed by 
Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c21882c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c21882c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c21882c1

Branch: refs/heads/yarn-native-services
Commit: c21882c1197c7bba06bdf26a73514f73f5f2104c
Parents: bb40b66
Author: Jian He 
Authored: Mon Sep 12 10:44:39 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../org/apache/slider/client/SliderClient.java  |  9 --
 .../apache/slider/core/zk/ZKIntegration.java| 34 +---
 2 files changed, 36 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c21882c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 5096bb7..fe4f1d2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -604,11 +604,14 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   BlockingZKWatcher watcher = new BlockingZKWatcher();
   client = ZKIntegration.newInstance(registryQuorum, user, clusterName, 
true, false, watcher,
   ZKIntegration.SESSION_TIMEOUT);
-  client.init();
-  watcher.waitForZKConnection(2 * 1000);
+  boolean fromCache = client.init();
+  if (!fromCache) {
+watcher.waitForZKConnection(2 * 1000);
+  }
 } catch (InterruptedException e) {
   client = null;
-  log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
+  log.warn("Interrupted - unable to connect to zookeeper quorum {}",
+  registryQuorum, e);
 } catch (IOException e) {
   log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c21882c1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index ca41e4b..6ed58d5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -33,6 +33,8 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 
@@ -65,6 +67,8 @@ public class ZKIntegration implements Watcher, Closeable {
   private final String clustername;
   private final String userPath;
   private int sessionTimeout = SESSION_TIMEOUT;
+  private static final Map ZK_SESSIONS = new HashMap<>();
+
 /**
  flag to set to indicate that the user path should be created if
  it is not already there
@@ -93,10 +97,32 @@ public class ZKIntegration implements Watcher, Closeable {
 this.userPath = mkSliderUserPath(username);
   }
 
-  public void init() throws IOException {
-assert zookeeper == null;
-log.debug("Binding ZK client to {}", zkConnection);
-zookeeper = new ZooKeeper(zkConnection, sessionTimeout, this, 
canBeReadOnly);
+  /**
+   * Returns true only if an active ZK session is available and retrieved from
+   * cache, false when it has to create a new one.
+   *
+   * @return true if from cache, false when new session created
+   * @throws IOException
+   */
+  public 

[46/63] [abbrv] hadoop git commit: YARN-5689. Update native services REST API to use agentless docker provider. Contributed by Billie Rinaldi & Gour Saha

[jianhe]

YARN-5689. Update native services REST API to use agentless docker provider. 
Contributed by Billie Rinaldi & Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ace62eb8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ace62eb8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ace62eb8

Branch: refs/heads/yarn-native-services
Commit: ace62eb8dd0ca0abbc9c31a8f277fda76d78a0d2
Parents: b0f0169
Author: Jian He 
Authored: Thu Oct 13 11:34:58 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../api/impl/ApplicationApiService.java | 251 ++-
 .../yarn/services/utils/RestApiConstants.java   |   3 -
 .../api/impl/TestApplicationApiService.java |   6 +-
 3 files changed, 79 insertions(+), 181 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ace62eb8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 9645696..0a62629 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,7 +50,6 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -68,10 +67,12 @@ import 
org.apache.hadoop.yarn.services.resource.Configuration;
 import org.apache.hadoop.yarn.services.resource.Container;
 import org.apache.hadoop.yarn.services.resource.ContainerState;
 import org.apache.hadoop.yarn.services.resource.Resource;
+import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.StateValues;
 import org.apache.slider.client.SliderClient;
 import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.params.ActionCreateArgs;
 import org.apache.slider.common.params.ActionFlexArgs;
 import org.apache.slider.common.params.ActionFreezeArgs;
@@ -88,12 +89,11 @@ import org.apache.slider.core.exceptions.NotFoundException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.providers.docker.DockerKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
@@ -211,7 +211,8 @@ public class ApplicationApiService implements 
ApplicationApi {
   application.setConfiguration(new Configuration());
 }
 addPropertyToConfiguration(application.getConfiguration(),
-PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+SliderKeys.COMPONENT_TYPE_KEY,
+SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
   }
   // resource
   validateApplicationResource(application.getResource(), null, application
@@ -249,7 +250,8 @@ public class ApplicationApiService implements 
ApplicationApi {
 comp.setConfiguration(new Configuration());
   }
   addPropertyToConfiguration(comp.getConfiguration(),
-  PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+  SliderKeys.COMPONENT_TYPE_KEY,
+  SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
   compNameArtifactIdMap.put(comp.getName(), 
comp.getArtifact().getId());
   comp.setName(comp.getArtifact().getId());
 }
@@ -339,9 +341,9 @@ public class ApplicationApiService implements 
ApplicationApi {
 final ActionCreateArgs createArgs = new ActionCreateArgs();
 addAppConfOptions(createArgs, application, compNameArtifactIdMap);
 

[23/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
new file mode 100644
index 000..c3a1d0e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
@@ -0,0 +1,62 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+
+/**
+ * An interface which services can implement to have their
+ * execution managed by the ServiceLauncher.
+ * The command line options will be passed down before the 
+ * {@link Service#init(Configuration)} operation is invoked via an
+ * invocation of {@link RunService#bindArgs(Configuration, String...)}
+ * After the service has been successfully started via {@link Service#start()}
+ * the {@link RunService#runService()} method is called to execute the 
+ * service. When this method returns, the service launcher will exit, using
+ * the return code from the method as its exit option.
+ */
+public interface RunService extends Service {
+
+  /**
+   * Propagate the command line arguments.
+   * This method is called before {@link Service#init(Configuration)};
+   * the configuration that is returned from this operation
+   * is the one that is passed on to the init operation.
+   * This permits implemenations to change the configuration before
+   * the init operation.n
+   * 
+   *
+   * @param config the initial configuration build up by the
+   * service launcher.
+   * @param args argument list list of arguments passed to the command line
+   * after any launcher-specific commands have been stripped.
+   * @return the configuration to init the service with. This MUST NOT be null.
+   * Recommended: pass down the config parameter with any changes
+   * @throws Exception any problem
+   */
+  Configuration bindArgs(Configuration config, String... args) throws 
Exception;
+  
+  /**
+   * Run a service. This called after {@link Service#start()}
+   * @return the exit code
+   * @throws Throwable any exception to report
+   */
+  int runService() throws Throwable ;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
new file mode 100644
index 000..27813b7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
@@ -0,0 +1,73 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed 

[52/63] [abbrv] hadoop git commit: YARN-5513. Move Java only tests from slider develop to yarn-native-services. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56715f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
new file mode 100644
index 000..7fceac7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.publisher;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.providers.agent.AgentProviderService;
+import org.apache.slider.server.appmaster.actions.QueueAccess;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public class TestAgentProviderService extends AgentProviderService {
+  protected static final Logger log =
+  LoggerFactory.getLogger(TestAgentProviderService.class);
+
+  public TestAgentProviderService() {
+super();
+log.info("TestAgentProviderService created");
+  }
+
+  @Override
+  public void bind(StateAccessForProviders stateAccessor,
+  QueueAccess queueAccess,
+  List liveContainers) {
+super.bind(stateAccessor, queueAccess, liveContainers);
+Map dummyProps = new HashMap();
+dummyProps.put("prop1", "val1");
+dummyProps.put("prop2", "val2");
+log.info("publishing dummy-site.xml with values {}", dummyProps);
+publishApplicationInstanceData("dummy-site", "dummy configuration",
+   dummyProps.entrySet());
+// publishing global config for testing purposes
+publishApplicationInstanceData("global", "global configuration",
+   stateAccessor.getAppConfSnapshot()
+   .getGlobalOptions().entrySet());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/56715f7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
new file mode 100644
index 000..f49e15a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestSliderProviderFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * 

[21/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
new file mode 100644
index 000..bf71861
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
+import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.slider.core.exceptions.ExceptionConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Class to bond to a Jersey client, for UGI integration and SPNEGO.
+ * 
+ *   Usage: create an instance, then when creating a Jersey Client
+ *   pass in to the constructor the handler provided by {@link #getHandler()}
+ *
+ * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
+ */
+public class UgiJerseyBinding implements
+HttpURLConnectionFactory {
+  private static final Logger log =
+  LoggerFactory.getLogger(UgiJerseyBinding.class);
+
+  private final UrlConnectionOperations operations;
+  private final URLConnectionClientHandler handler;
+
+  /**
+   * Construct an instance
+   * @param operations operations instance
+   */
+  @SuppressWarnings("ThisEscapedInObjectConstruction")
+  public UgiJerseyBinding(UrlConnectionOperations operations) {
+Preconditions.checkArgument(operations != null, "Null operations");
+this.operations = operations;
+handler = new URLConnectionClientHandler(this);
+  }
+
+  /**
+   * Create an instance off the configuration. The SPNEGO policy
+   * is derived from the current UGI settings.
+   * @param conf config
+   */
+  public UgiJerseyBinding(Configuration conf) {
+this(new UrlConnectionOperations(conf));
+  }
+
+  /**
+   * Get a URL connection. 
+   * @param url URL to connect to
+   * @return the connection
+   * @throws IOException any problem. {@link AuthenticationException} 
+   * errors are wrapped
+   */
+  @Override
+  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+try {
+  // open a connection handling status codes and so redirections
+  // but as it opens a connection, it's less useful than you think.
+
+  return operations.openConnection(url);
+} catch (AuthenticationException e) {
+  throw new IOException(e);
+}
+  }
+
+  public UrlConnectionOperations getOperations() {
+return operations;
+  }
+
+  public URLConnectionClientHandler getHandler() {
+return handler;
+  }
+  
+  /**
+   * Get the SPNEGO flag (as found in the operations instance
+   * @return the spnego policy
+   */
+  public boolean isUseSpnego() {
+return operations.isUseSpnego();
+  }
+
+
+  /**
+   * Uprate error codes 400 and up into faults; 
+   * 
+   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
+   */
+  public static 

[25/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
new file mode 100644
index 000..86025ee
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Locale;
+import java.util.Properties;
+
+/**
+ * Extract the version properties, which will look something like
+ * 
+ * application.name=${pom.name}
+ * application.version=${pom.version}
+ * application.build=${buildNumber}
+ * application.build.java.version=${java.version}
+ * application.build.info=${pom.name}-${pom.version} Built against 
${buildNumber} on ${java.version} by ${user.name}
+ * 
+ * 
+ * the mvn process-resources target will expand the properties
+ * and add the resources to target/classes, which will then look something like
+ * 
+ *   application.name=Slider Core
+ *   application.version=0.7.1-SNAPSHOT
+ *   application.build=1dd69
+ *   application.build.java.version=1.7.0_45
+ *   application.build.user=stevel
+ *   application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 
1.7.0_45 by stevel
+ * 
+ * 
+ * Note: the values will change and more properties added.
+ */
+public class SliderVersionInfo {
+  private static final Logger log = 
LoggerFactory.getLogger(SliderVersionInfo.class);
+
+  /**
+   * Name of the resource containing the filled-in-at-runtime props
+   */
+  public static final String VERSION_RESOURCE =
+  "org/apache/slider/providers/dynamic/application.properties";
+
+  public static final String APP_NAME = "application.name";
+  public static final String APP_VERSION = "application.version";
+  public static final String APP_BUILD = "application.build";
+  public static final String APP_BUILD_JAVA_VERSION = 
"application.build.java.version";
+  public static final String APP_BUILD_USER = "application.build.user";
+  public static final String APP_BUILD_INFO = "application.build.info";
+  public static final String HADOOP_BUILD_INFO = "hadoop.build.info";
+  public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info";
+
+
+  public static Properties loadVersionProperties()  {
+Properties props = new Properties();
+URL resURL = SliderVersionInfo.class.getClassLoader()
+   .getResource(VERSION_RESOURCE);
+assert resURL != null : "Null resource " + VERSION_RESOURCE;
+
+try {
+  InputStream inStream = resURL.openStream();
+  assert inStream != null : "Null input stream from " + VERSION_RESOURCE;
+  props.load(inStream);
+} catch (IOException e) {
+  log.warn("IOE loading " + VERSION_RESOURCE, e);
+}
+return props;
+  }
+
+  /**
+   * Load the version info and print it
+   * @param logger logger
+   */
+  public static void loadAndPrintVersionInfo(Logger logger) {
+Properties props = loadVersionProperties();
+logger.info(props.getProperty(APP_BUILD_INFO));
+logger.info("Compiled against Hadoop {}",
+props.getProperty(HADOOP_BUILD_INFO));
+logger.info(getHadoopVersionString());
+  }
+  
+  public static String getHadoopVersionString() {
+return String.format(Locale.ENGLISH,
+"Hadoop runtime version %s with source checksum %s and build date %s",
+VersionInfo.getBranch(),
+

[56/63] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6698c170/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
new file mode 100644
index 000..a3780cc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or application. The resource specified at
+ * the app (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu  memory are exepected. It raises a validation
+ * exception otherwise.
+ **/
+
+@ApiModel(description = "Resource determines the amount of resources (vcores, 
memory, network, etc.) usable by a container. This field determines the 
resource to be applied for all the containers of a component or application. 
The resource specified at the app (or global) level can be overriden at the 
component level. Only one of profile OR cpu & memory are exepected. It raises a 
validation exception otherwise.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource {
+  private static final long serialVersionUID = -6431667797380250037L;
+
+  private String profile = null;
+  private Integer cpus = null;
+  private String memory = null;
+
+  /**
+   * Each resource profile has a unique id which is associated with a
+   * cluster-level predefined memory, cpus, etc.
+   **/
+  public Resource profile(String profile) {
+this.profile = profile;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Each resource profile has a 
unique id which is associated with a cluster-level predefined memory, cpus, 
etc.")
+  @JsonProperty("profile")
+  public String getProfile() {
+return profile;
+  }
+
+  public void setProfile(String profile) {
+this.profile = profile;
+  }
+
+  /**
+   * Amount of vcores allocated to each container (optional but overrides cpus
+   * in profile if specified).
+   **/
+  public Resource cpus(Integer cpus) {
+this.cpus = cpus;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Amount of vcores allocated to 
each container (optional but overrides cpus in profile if specified).")
+  @JsonProperty("cpus")
+  public Integer getCpus() {
+return cpus;
+  }
+
+  public void setCpus(Integer cpus) {
+this.cpus = cpus;
+  }
+
+  /**
+   * Amount of memory allocated to each container (optional but overrides 
memory
+   * in profile if specified). Currently accepts only an integer value and
+   * default unit is in MB.
+   **/
+  public Resource memory(String memory) {
+this.memory = memory;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Amount of memory allocated to 
each container (optional but overrides memory in profile if specified). 
Currently accepts only an integer value and default unit is in MB.")
+  @JsonProperty("memory")
+  public String getMemory() {
+return memory;
+  }
+
+  public void setMemory(String memory) {
+this.memory = memory;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != 

[26/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
new file mode 100644
index 000..73e0879
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -0,0 +1,2548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.Slider;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import 

[30/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
new file mode 100644
index 000..5758f79
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+  /**
+   * starting point for exit codes; not an exception itself
+   */
+  int _EXIT_CODE_BASE =   64;
+
+  /**
+   * service entered the failed state: {@value}
+   */
+  int EXIT_YARN_SERVICE_FAILED =  65;
+
+  /**
+   * service was killed: {@value}
+   */
+  int EXIT_YARN_SERVICE_KILLED =  66;
+
+  /**
+   * timeout on monitoring client: {@value}
+   */
+  int EXIT_TIMED_OUT =67;
+
+  /**
+   * service finished with an error: {@value}
+   */
+  int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
+
+  /**
+   * the application instance is unknown: {@value}
+   */
+  int EXIT_UNKNOWN_INSTANCE = 69;
+
+  /**
+   * the application instance is in the wrong state for that operation: 
{@value}
+   */
+  int EXIT_BAD_STATE =70;
+
+  /**
+   * A spawned master process failed 
+   */
+  int EXIT_PROCESS_FAILED =   71;
+
+  /**
+   * The instance failed -too many containers were
+   * failing or some other threshold was reached
+   */
+  int EXIT_DEPLOYMENT_FAILED =72;
+
+  /**
+   * The application is live -and the requested operation
+   * does not work if the cluster is running
+   */
+  int EXIT_APPLICATION_IN_USE =   73;
+
+  /**
+   * There already is an application instance of that name
+   * when an attempt is made to create a new instance
+   */
+  int EXIT_INSTANCE_EXISTS =  75;
+
+  /**
+   * Exit code when the configurations in valid/incomplete: {@value}
+   */
+  int EXIT_BAD_CONFIGURATION =77;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
new file mode 100644
index 000..ba3effc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  

[16/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
new file mode 100644
index 000..67d3647
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.appmaster.PublishedArtifacts;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+
+/**
+ * Exists just to move some functionality out of AppMaster into a peer class
+ * of the actual service provider doing the real work
+ */
+public class SliderAMProviderService extends AbstractProviderService implements
+ProviderCore,
+AgentKeys,
+SliderKeys {
+
+  public SliderAMProviderService() {
+super("SliderAMProviderService");
+  }
+
+  @Override
+  public String getHumanName() {
+return "Slider Application";
+  }
+  
+  @Override
+  public Configuration loadProviderConfigurationInformation(File confDir) 
throws
+  BadCommandArgumentsException,
+  IOException {
+return null;
+  }
+
+  @Override
+  public void buildContainerLaunchContext(ContainerLauncher containerLauncher,
+  AggregateConf instanceDefinition,
+  Container container,
+  ProviderRole role,
+  SliderFileSystem sliderFileSystem,
+  Path generatedConfPath,
+  MapOperations resourceComponent,
+  MapOperations appComponent,
+  Path containerTmpDirPath) throws IOException, SliderException {
+  }
+
+  @Override
+  public List getRoles() {
+return new ArrayList<>(0);
+  }
+
+  @Override
+  public void validateInstanceDefinition(AggregateConf instanceDefinition) 
throws
+  SliderException {
+
+  }
+
+  @Override
+  public void applyInitialRegistryDefinitions(URL amWebURI,
+  URL agentOpsURI,
+  URL agentStatusURI,
+  ServiceRecord serviceRecord)
+  throws IOException {
+super.applyInitialRegistryDefinitions(amWebURI,
+agentOpsURI,
+agentStatusURI,
+

[07/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
new file mode 100644
index 000..781ae00
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import com.google.gson.annotations.SerializedName;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class AgentEnv {
+
+  /**
+   * Various directories, configurable in ambari-agent.ini
+   */
+  private Directory[] stackFoldersAndFiles = new Directory[0];
+
+  /**
+   * Directories that match name /etc/alternatives/*conf
+   */
+  private Alternative[] alternatives = new Alternative[0];
+
+  /**
+   * List of existing users
+   */
+  private ExistingUser[] existingUsers = new ExistingUser[0];
+
+  /**
+   * List of repos
+   */
+  private String[] existingRepos = new String[0];
+
+  /**
+   * List of packages
+   */
+  private PackageDetail[] installedPackages = new PackageDetail[0];
+
+  /**
+   * The host health report
+   */
+  private HostHealth hostHealth = new HostHealth();
+
+  private Integer umask;
+
+  private Boolean iptablesIsRunning;
+
+  public Integer getUmask() {
+return umask;
+  }
+
+  public void setUmask(Integer umask) {
+this.umask = umask;
+  }
+
+  public Directory[] getStackFoldersAndFiles() {
+return stackFoldersAndFiles;
+  }
+
+  public void setStackFoldersAndFiles(Directory[] dirs) {
+stackFoldersAndFiles = dirs;
+  }
+
+  public void setExistingUsers(ExistingUser[] users) {
+existingUsers = users;
+  }
+
+  public ExistingUser[] getExistingUsers() {
+return existingUsers;
+  }
+
+  public void setAlternatives(Alternative[] dirs) {
+alternatives = dirs;
+  }
+
+  public Alternative[] getAlternatives() {
+return alternatives;
+  }
+
+  public void setExistingRepos(String[] repos) {
+existingRepos = repos;
+  }
+
+  public String[] getExistingRepos() {
+return existingRepos;
+  }
+
+  public void setInstalledPackages(PackageDetail[] packages) {
+installedPackages = packages;
+  }
+
+  public PackageDetail[] getInstalledPackages() {
+return installedPackages;
+  }
+
+  public void setHostHealth(HostHealth healthReport) {
+hostHealth = healthReport;
+  }
+
+  public HostHealth getHostHealth() {
+return hostHealth;
+  }
+
+  public Boolean getIptablesIsRunning() {
+return iptablesIsRunning;
+  }
+
+  public void setIptablesIsRunning(Boolean iptablesIsRunning) {
+this.iptablesIsRunning = iptablesIsRunning;
+  }
+
+  public static class HostHealth {
+/**
+ * Java processes running on the system.  Default empty array.
+ */
+@SerializedName("activeJavaProcs")
+private JavaProc[] activeJavaProcs = new JavaProc[0];
+
+/**
+ * The current time when agent send the host check report
+ */
+@SerializedName("agentTimeStampAtReporting")
+private long agentTimeStampAtReporting = 0;
+
+/**
+ * The current time when host check report was received
+ */
+@SerializedName("serverTimeStampAtReporting")
+private long serverTimeStampAtReporting = 0;
+
+/**
+ * Live services running on the agent
+ */
+@SerializedName("liveServices")
+private LiveService[] liveServices = new LiveService[0];
+
+   

[18/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
new file mode 100644
index 000..281895a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AgentRoles {
+
+  /**
+   * List of roles Agent provider does not have any roles by default. All 
roles are read from the application
+   * specification.
+   */
+  protected static final List ROLES =
+  new ArrayList();
+
+  public static List getRoles() {
+return ROLES;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
new file mode 100644
index 000..cfcfc5d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import 
org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
+import 
org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
+import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
+import 
org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class AgentUtils {
+  private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
+
+  

[12/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
new file mode 100644
index 000..dd4785d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.rpc;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcEngine;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.ServiceNotReadyException;
+import org.apache.slider.core.exceptions.SliderException;
+
+import static org.apache.slider.common.SliderXmlConfKeys.*; 
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+public class RpcBinder {
+  protected static final Logger log =
+LoggerFactory.getLogger(RpcBinder.class);
+
+  /**
+   * Create a protobuf server bonded to the specific socket address
+   * @param addr address to listen to; 0.0.0.0 as hostname acceptable
+   * @param conf config
+   * @param secretManager token secret handler
+   * @param numHandlers threads to service requests
+   * @param blockingService service to handle
+   * @param portRangeConfig range of ports
+   * @return the IPC server itself
+   * @throws IOException
+   */
+  public static Server createProtobufServer(InetSocketAddress addr,
+Configuration conf,
+SecretManager secretManager,
+int numHandlers,
+BlockingService blockingService,
+String portRangeConfig) throws
+  IOException {
+Class sliderClusterAPIClass = registerSliderAPI(
+conf);
+RPC.Server server = new 
RPC.Builder(conf).setProtocol(sliderClusterAPIClass)
+ .setInstance(blockingService)
+   

[47/63] [abbrv] hadoop git commit: YARN-5813. Slider should not try to set a negative lifetime timeout value. Contributed by Jian He

[jianhe]

YARN-5813. Slider should not try to set a negative lifetime timeout value. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc50c70a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc50c70a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc50c70a

Branch: refs/heads/yarn-native-services
Commit: dc50c70a8cf9fcfea862e17a5a63af022ca9
Parents: 60ee3f5
Author: Gour Saha 
Authored: Tue Nov 1 17:39:54 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../src/main/java/org/apache/slider/client/SliderClient.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc50c70a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d1f88c5..ea10ed0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2120,7 +2120,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 amLauncher.setKeepContainersOverRestarts(true);
 // set lifetime in submission context;
 Map appTimeout = new HashMap<>();
-appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+if (lifetime >= 0) {
+  appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+}
 amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
 int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
 amLauncher.setMaxAppAttempts(maxAppAttempts);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[39/63] [abbrv] hadoop git commit: YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha

[jianhe]

YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7616537
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7616537
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7616537

Branch: refs/heads/yarn-native-services
Commit: f76165379a6e509a1eccfc8ba7d1b59fff29310e
Parents: 6698c17
Author: Jian He 
Authored: Wed Oct 12 13:27:53 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 416 +++
 1 file changed, 416 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7616537/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
new file mode 100644
index 000..6169fcd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -0,0 +1,416 @@
+# Hadoop YARN REST APIs for services v1 spec in YAML
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+swagger: '2.0'
+info:
+  title: "[YARN-4793] Simplified API layer for services and beyond"
+  description: |
+Bringing a new service on YARN today is not a simple experience. The APIs 
of existing frameworks are either too low level (native YARN), require writing 
new code (for frameworks with programmatic APIs) or writing a complex spec (for 
declarative frameworks). In addition to building critical building blocks 
inside YARN (as part of other efforts at 
link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a 
need for simplifying the user facing story for building services. Experience of 
projects like Apache Slider running real-life services like HBase, Storm, 
Accumulo, Solr etc, gives us some very good insights on how simplified APIs for 
services should look like.
+
+
+To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component apps to 
complex multi-component assemblies needing orchestration.
+
+
+We should also look at making this a unified REST based entry point for 
other important features like resource-profile management 
(link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), 
package-definitions' lifecycle-management and service-discovery 
(link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]).
 We also need to flesh out its relation to our present much lower level REST 
APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN 
for application-submission and management.
+
+
+This document spotlights on this specification. In most of the cases, the 
application owner will not be forced to make any changes to their application. 
This is primarily true if the application is packaged with containerization 
technologies like docker. Irrespective of how complex the application is, there 
will be hooks provided at appropriate layers to allow pluggable and 
customizable application behavior.
+
+  version: "1.0.0"
+  license:
+name: Apache 2.0
+url: http://www.apache.org/licenses/LICENSE-2.0.html
+# the domain of the service
+host: host.mycompany.com
+# array of all schemes that your API supports
+schemes:
+  - http
+  - https
+# will be prefixed to all paths

[43/63] [abbrv] hadoop git commit: YARN-5680. Add 2 new fields in Slider status output - image-name and is-privileged-container. Contributed by Billie Rinaldi

[jianhe]

YARN-5680. Add 2 new fields in Slider status output - image-name and 
is-privileged-container. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/686c663c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/686c663c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/686c663c

Branch: refs/heads/yarn-native-services
Commit: 686c663c3cbbf9b246e2d9564a95890e9d48d67a
Parents: 5c2a782
Author: Gour Saha 
Authored: Tue Oct 25 20:00:27 2016 -0700
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../org/apache/slider/providers/docker/DockerKeys.java  |  1 +
 .../slider/providers/docker/DockerProviderService.java  |  2 +-
 .../apache/slider/server/appmaster/state/AppState.java  | 12 
 3 files changed, 14 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/686c663c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
index 40b73a2..0e1d288 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
@@ -26,6 +26,7 @@ public interface DockerKeys {
   String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
 
   String DEFAULT_DOCKER_NETWORK = "bridge";
+  Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
 
   String OUT_FILE = "stdout.txt";
   String ERR_FILE = "stderr.txt";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/686c663c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index af36620..cc319ee 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -131,7 +131,7 @@ public class DockerProviderService extends 
AbstractProviderService implements
 launcher.setDockerNetwork(appConf.getComponentOpt(roleGroup, 
DOCKER_NETWORK,
 DEFAULT_DOCKER_NETWORK));
 launcher.setRunPrivilegedContainer(appConf.getComponentOptBool(roleGroup,
-DOCKER_USE_PRIVILEGED, false));
+DOCKER_USE_PRIVILEGED, DEFAULT_DOCKER_USE_PRIVILEGED));
 
 // Set the environment
 Map standardTokens = providerUtils.getStandardTokenMap(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/686c663c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 49e7b78..6db375d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -91,6 +91,9 @@ 

[13/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
new file mode 100644
index 000..37a8935
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.health.HealthCheckRegistry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Class for all metrics and monitoring
+ */
+public class MetricsAndMonitoring extends CompositeService {
+  protected static final Logger log =
+LoggerFactory.getLogger(MetricsAndMonitoring.class);
+  public MetricsAndMonitoring(String name) {
+super(name);
+  }
+  
+  public MetricsAndMonitoring() {
+super("MetricsAndMonitoring");
+  }
+  
+  /**
+   * Singleton of metrics registry
+   */
+  final MetricRegistry metrics = new MetricRegistry();
+
+  final HealthCheckRegistry health = new HealthCheckRegistry();
+
+  private final Map meterAndCounterMap
+  = new ConcurrentHashMap<>();
+
+  private final List metricSets = new ArrayList<>();
+
+  /**
+   * List of recorded events
+   */
+  private final List eventHistory = new ArrayList<>(100);
+
+  public static final int EVENT_LIMIT = 1000;
+
+  public MetricRegistry getMetrics() {
+return metrics;
+  }
+
+  public HealthCheckRegistry getHealth() {
+return health;
+  }
+
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+addService(new MetricsBindingService("MetricsBindingService",
+metrics));
+super.serviceInit(conf);
+  }
+
+  @Override
+  protected void serviceStop() throws Exception {
+super.serviceStop();
+for (MetricSet set : metricSets) {
+  unregister(set);
+}
+  }
+
+  public MeterAndCounter getMeterAndCounter(String name) {
+return meterAndCounterMap.get(name);
+  }
+
+  /**
+   * Get or create the meter/counter pair
+   * @param name name of instance
+   * @return an instance
+   */
+  public MeterAndCounter getOrCreateMeterAndCounter(String name) {
+MeterAndCounter instance = meterAndCounterMap.get(name);
+if (instance == null) {
+  synchronized (this) {
+// check in a sync block
+instance = meterAndCounterMap.get(name);
+if (instance == null) {
+  instance = new MeterAndCounter(metrics, name);
+  meterAndCounterMap.put(name, instance);
+}
+  }
+}
+return instance;
+  }
+
+  /**
+   * Get a specific meter and mark it. This will create and register it on 
demand.
+   * @param name name of meter/counter
+   */
+  public void markMeterAndCounter(String name) {
+MeterAndCounter meter = getOrCreateMeterAndCounter(name);
+meter.mark();
+  }
+
+  /**
+   * Given a {@link Metric}, registers it under the given name.
+   *
+   * @param name   the name of the metric
+   * @param metric the metric
+   * @param the type of the metric
+   * @return {@code metric}
+   * @throws IllegalArgumentException if the name is already registered
+   */
+  public  T register(String name, 

[29/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
new file mode 100644
index 000..739b5fc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_LIST},
+commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionListArgs extends AbstractActionArgs {
+  @Override
+  public String getActionName() {
+return SliderActions.ACTION_LIST;
+  }
+
+  @Parameter(names = {ARG_LIVE},
+  description = "List only live application instances")
+  public boolean live;
+
+  @Parameter(names = {ARG_STATE},
+  description = "list only applications in the specific YARN state")
+  public String state = "";
+  
+  @Parameter(names = {ARG_VERBOSE},
+  description = "print out information in details")
+  public boolean verbose = false;
+
+  @Parameter(names = {ARG_CONTAINERS},
+  description = "List containers of an application instance")
+  public boolean containers;
+
+  @Parameter(names = {ARG_VERSION},
+  description = "Filter containers by app version (used with " +
+ARG_CONTAINERS + ")")
+  public String version;
+
+  @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
+  description = "Filter containers by component names (used with " +
+ARG_CONTAINERS + ")")
+  public Set components = new HashSet<>(0);
+
+  /**
+   * Get the min #of params expected
+   * @return the min number of params in the {@link #parameters} field
+   */
+  public int getMinParams() {
+return 0;
+  }
+
+  @Override
+  public int getMaxParams() {
+return 1;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
new file mode 100644
index 000..1b73522
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the 

[08/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
new file mode 100644
index 000..ad91183
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The methods to offer state access to the providers and other parts of
+ * the system which want read-only access to the state.
+ */
+public interface StateAccessForProviders {
+
+  /**
+   * Get a map of role status entries by role Id
+   * @return the map of currently defined roles.
+   */
+  Map getRoleStatusMap();
+
+  /**
+   * Get the name of the application
+   * @return the name
+   */
+  String getApplicationName();
+
+  /**
+   * Get the published configurations
+   * @return the configuration set
+   */
+  PublishedConfigSet getPublishedSliderConfigurations();
+
+  /**
+   * Get the published exports set
+   * @return
+   */
+  PublishedExportsSet getPublishedExportsSet();
+
+  /**
+   * Get a named published config set
+   * @param name name to look up
+   * @return the instance or null
+   */
+  PublishedConfigSet getPublishedConfigSet(String name);
+
+  /**
+   * Get a named published config set, creating it if need be.
+   * @param name name to look up
+   * @return the instance -possibly a new one
+   */
+  PublishedConfigSet getOrCreatePublishedConfigSet(String name);
+
+  /**
+   * List the config sets -this takes a clone of the current set
+   * @return a list of config sets
+   */
+  List listConfigSets();
+
+  /**
+   * Get a map of all the failed containers
+   * @return map of recorded failed containers
+   */
+  Map getFailedContainers();
+
+  /**
+   * Get the live containers.
+   * 
+   * @return the live nodes
+   */
+  Map getLiveContainers();
+
+  /**
+   * Get the current cluster description 
+   * @return the actual state of the cluster
+   */
+  ClusterDescription getClusterStatus();
+
+  /**
+   * Get at the snapshot of the resource config
+   * Changes here do not affect the application state.
+   * @return the most recent settings
+   */
+  ConfTreeOperations getResourcesSnapshot();
+
+  /**
+   * Get at the snapshot of the appconf config
+   * Changes here do not affect the application state.
+   * @return the most recent settings
+   */
+  ConfTreeOperations getAppConfSnapshot();
+
+  /**
+   * Get at the snapshot of the internals config.
+   * Changes here do not 

[24/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
new file mode 100644
index 000..5a3eb3d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -0,0 +1,528 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.MapOperations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Launcher of applications: base class
+ */
+public abstract class AbstractLauncher extends Configured {
+  private static final Logger log =
+LoggerFactory.getLogger(AbstractLauncher.class);
+  public static final String CLASSPATH = "CLASSPATH";
+  /**
+   * Filesystem to use for the launch
+   */
+  protected final CoreFileSystem coreFileSystem;
+  /**
+   * Env vars; set up at final launch stage
+   */
+  protected final Map envVars = new HashMap<>();
+  protected final MapOperations env = new MapOperations("env", envVars);
+  protected final ContainerLaunchContext containerLaunchContext =
+Records.newRecord(ContainerLaunchContext.class);
+  protected final List commands = new ArrayList<>(20);
+  protected final Map localResources = new HashMap<>();
+  protected final Map mountPaths = new HashMap<>();
+  private final Map serviceData = new HashMap<>();
+  // security
+  protected final Credentials credentials;
+  protected LogAggregationContext logAggregationContext;
+  protected boolean yarnDockerMode = false;
+  protected String dockerImage;
+  protected String yarnContainerMountPoints;
+  protected String runPrivilegedContainer;
+
+
+  /**
+   * Create instance.
+   * @param conf configuration
+   * @param coreFileSystem filesystem
+   * @param credentials initial set of credentials -null is permitted
+   */
+  protected AbstractLauncher(Configuration conf,
+  CoreFileSystem coreFileSystem,
+  Credentials credentials) {
+super(conf);
+this.coreFileSystem = coreFileSystem;
+this.credentials = credentials != null ? credentials: new Credentials();
+  }
+
+  /**
+   * Get the container. Until "completed", this isn't valid to launch.
+   * @return the container to launch
+   */
+  public ContainerLaunchContext getContainerLaunchContext() {
+return 

[40/63] [abbrv] hadoop git commit: Modify pom file for slider

[jianhe]

Modify pom file for slider


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f030621
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f030621
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f030621

Branch: refs/heads/yarn-native-services
Commit: 2f0306218779c15681f180ab09f7b68325dfc31a
Parents: 12ee7a7
Author: Jian He 
Authored: Fri Aug 12 09:28:41 2016 +0800
Committer: Jian He 
Committed: Mon Nov 28 13:09:08 2016 -0800

--
 .../hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml | 2 +-
 .../hadoop-yarn-applications/hadoop-yarn-slider/pom.xml| 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f030621/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 14130c5..591a5ca 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -18,7 +18,7 @@
   4.0.0
   
 org.apache.hadoop
-hadoop-yarn-applications
+hadoop-yarn-slider
 3.0.0-alpha2-SNAPSHOT
   
   org.apache.hadoop

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f030621/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
index ad374f8d..780f68b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
@@ -17,7 +17,7 @@
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
   http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
-hadoop-yarn
+hadoop-yarn-applications
 org.apache.hadoop
 3.0.0-alpha2-SNAPSHOT
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[19/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
new file mode 100644
index 000..4ffae7c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -0,0 +1,3212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.ConfigUtils;
+import org.apache.slider.core.registry.docstore.ExportEntry;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.MonitorDetail;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import 
org.apache.slider.providers.agent.application.metadata.AbstractComponent;
+import org.apache.slider.providers.agent.application.metadata.Application;
+import 

[15/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
new file mode 100644
index 000..b767059
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -0,0 +1,2450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.health.HealthCheckRegistry;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.BlockingService;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
+
+import 
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;

[32/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
new file mode 100644
index 000..8210f4d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -0,0 +1,4569 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.Files;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.KerberosDiags;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.SliderApplicationApi;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.StateValues;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformationList;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.client.ipc.SliderApplicationIpcClient;
+import org.apache.slider.client.ipc.SliderClusterOperations;
+import org.apache.slider.common.Constants;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;

[11/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
new file mode 100644
index 000..16c2435
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -0,0 +1,2489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterDescriptionKeys;
+import org.apache.slider.api.ClusterDescriptionOperations;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.persist.AggregateConfSerDeser;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.LongGauge;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.management.MetricsConstants;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import 

[10/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
new file mode 100644
index 000..a8aa1a2
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Binding information for application states; designed to be extensible
+ * so that tests don't have to be massivley reworked when new arguments
+ * are added.
+ */
+public class AppStateBindingInfo {
+  public AggregateConf instanceDefinition;
+  public Configuration serviceConfig = new Configuration();
+  public Configuration publishedProviderConf = new Configuration(false);
+  public List roles = new ArrayList<>();
+  public FileSystem fs;
+  public Path historyPath;
+  public List liveContainers = new ArrayList<>(0);
+  public Map applicationInfo = new HashMap<>();
+  public ContainerReleaseSelector releaseSelector = new 
SimpleReleaseSelector();
+  /** node reports off the RM. */
+  public List nodeReports = new ArrayList<>(0);
+
+  public void validate() throws IllegalArgumentException {
+Preconditions.checkArgument(instanceDefinition != null, "null 
instanceDefinition");
+Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig");
+Preconditions.checkArgument(publishedProviderConf != null, "null 
publishedProviderConf");
+Preconditions.checkArgument(releaseSelector != null, "null 
releaseSelector");
+Preconditions.checkArgument(roles != null, "null providerRoles");
+Preconditions.checkArgument(fs != null, "null fs");
+Preconditions.checkArgument(historyPath != null, "null historyDir");
+Preconditions.checkArgument(nodeReports != null, "null nodeReports");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
new file mode 100644
index 000..5b3a93c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF 

[17/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
new file mode 100644
index 000..78bb8c1
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Component defined in master package metainfo.json
+ */
+public class Component extends AbstractComponent {
+
+  String category = CATEGORY_MASTER;
+  String publishConfig = Boolean.FALSE.toString();
+  String minInstanceCount = "0";
+  String maxInstanceCount;
+  String autoStartOnFailure = Boolean.FALSE.toString();
+  String appExports;
+  String compExports;
+  String type = TYPE_STANDARD;
+  List componentExports = new ArrayList<>();
+  List dockerContainers = new ArrayList<>();
+  List configFiles = new ArrayList<>();
+
+  public Component() {
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public void setType(String type) {
+this.type = type;
+  }
+
+  public String getCategory() {
+return category;
+  }
+
+  public void setCategory(String category) {
+this.category = category;
+  }
+
+  public String getPublishConfig() {
+return publishConfig;
+  }
+
+  public void setPublishConfig(String publishConfig) {
+this.publishConfig = publishConfig;
+  }
+
+  public String getAutoStartOnFailure() {
+return autoStartOnFailure;
+  }
+
+  public void setAutoStartOnFailure(String autoStartOnFailure) {
+this.autoStartOnFailure = autoStartOnFailure;
+  }
+
+  public String getAppExports() {
+return appExports;
+  }
+
+  public void setAppExports(String appExports) {
+this.appExports = appExports;
+  }
+
+  public String getCompExports() {
+return compExports;
+  }
+
+  public void setCompExports(String compExports) {
+this.compExports = compExports;
+  }
+
+  public String getMinInstanceCount() {
+return minInstanceCount;
+  }
+  
+  @JsonProperty("dockerContainers")
+  public List getDockerContainers() {
+ return this.dockerContainers;
+  }
+  
+  public Boolean getAutoStartOnFailureBoolean() {
+if (SliderUtils.isUnset(getAutoStartOnFailure())) {
+  return Boolean.FALSE;
+}
+
+return Boolean.parseBoolean(getAutoStartOnFailure());
+  }
+
+  public int getMinInstanceCountInt() throws BadConfigException {
+if (SliderUtils.isUnset(minInstanceCount)) {
+  return 0;
+}
+
+try {
+  return Integer.parseInt(minInstanceCount);
+} catch (NumberFormatException nfe) {
+  throw new BadConfigException(nfe, "Invalid value for minInstanceCount 
for %s", name);
+}
+  }
+
+  public int getMaxInstanceCountInt() throws BadConfigException {
+if (SliderUtils.isUnset(maxInstanceCount)) {
+  return Integer.MAX_VALUE;
+}
+
+try {
+  return Integer.parseInt(maxInstanceCount);
+} catch (NumberFormatException nfe) {
+  throw new BadConfigException(nfe, "Invalid value for maxInstanceCount 
for %s", name);
+}
+  }
+
+  public void setMinInstanceCount(String minInstanceCount) {
+this.minInstanceCount = minInstanceCount;
+  }
+
+  public String getMaxInstanceCount() {
+return 

[22/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
new file mode 100644
index 000..1bdfb9c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Client code for interacting with a list of service instances.
+ * The initial logic just enumerates service instances in the YARN RM
+ */
+public class YarnAppListClient {
+
+  private final SliderYarnClientImpl yarnClient;
+  private final String username;
+  private final Configuration conf;
+  private static final Logger log =
+  LoggerFactory.getLogger(YarnAppListClient.class);
+
+  public YarnAppListClient(SliderYarnClientImpl yarnClient,
+  String username,
+  Configuration conf) {
+
+Preconditions.checkArgument(yarnClient != null,
+"yarn client is null: is app inited?");
+Preconditions.checkArgument(username != null,
+"username is null");
+Preconditions.checkArgument(conf != null,
+"conf parameter is null");
+this.yarnClient = yarnClient;
+this.username = username;
+this.conf = conf;
+  }
+
+  /**
+   * find all live instances of a specific app -if there is more than one 
+   * in the cluster, this returns them all. State should be running or earlier
+   * in the lifecycle
+   * @param appname application name
+   * @return the list of all matching application instances
+   */
+  public List findAllLiveInstances(String appname)
+throws YarnException, IOException {
+return yarnClient.findAllLiveInstances(username, appname);
+  }
+
+
+  /**
+   * Find an instance of a application belong to the current user
+   * @param appname application name
+   * @return the app report or null if none is found
+   * @throws YarnException YARN issues
+   * @throws IOException IO problems
+   */
+  public ApplicationReport findInstance(String appname) throws
+YarnException,
+IOException {
+List instances = listInstances(null);
+return yarnClient.findClusterInInstanceList(instances, appname);
+  }
+
+  /**
+   * List instances belonging to the specific user
+   * @return a possibly empty list of AMs
+   */
+  public List listInstances()
+  throws YarnException, IOException {
+return listInstances(null);
+  }
+
+  /**
+   * List instances belonging to a specific user
+   * @return a possibly empty list of AMs
+   * @param user user if not the default. null means default, "" means all 
users, 
+   * otherwise it is the name of a user
+   */
+  public List listInstances(String user)
+  throws YarnException, IOException {
+String listUser = user == null ? username : user;
+

[14/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
new file mode 100644
index 000..6b61681
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class ActionFlexCluster extends AsyncAction {
+
+  public final ConfTree resources;
+  
+  public ActionFlexCluster(String name,
+  long delay,
+  TimeUnit timeUnit, ConfTree resources) {
+super(name, delay, timeUnit, ATTR_CHANGES_APP_SIZE);
+this.resources = resources;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+  QueueAccess queueService,
+  AppState appState) throws Exception {
+appMaster.flexCluster(resources);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
new file mode 100644
index 000..e2ad559
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Exit an emergency JVM halt.
+ * @see ExitUtil#halt(int, String) 
+ */
+public class ActionHalt extends AsyncAction {
+
+  private final int status;
+  private final String text;
+
+  public ActionHalt(
+  int status,
+  String text,
+  long delay, TimeUnit timeUnit) {
+
+// do not declare that this action halts the cluster ... keep it a surprise
+super("Halt", delay, timeUnit);
+this.status = status;
+this.text = 

[03/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
new file mode 100644
index 000..e436ae9
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -0,0 +1,495 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import com.google.inject.Singleton;
+import org.apache.commons.io.FileUtils;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.text.MessageFormat;
+
+@Singleton
+public class CertificateManager {
+
+  private static final Logger LOG =
+  LoggerFactory.getLogger(CertificateManager.class);
+
+  private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " +
+  "-passout pass:{0} -out {1}" + File.separator + "{2} 4096 ";
+  private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " +
+  "-new -key {1}" + File.separator + "{2} -out {1}" + File.separator +
+  "{5} -config {1}" + File.separator + "ca.config " +
+  "-subj {6} -batch";
+  private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " +
+"-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator
++ "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator
++ "ca.config -batch -infiles {1}" + File.separator + "{5}";
+  private static final String EXPRT_KSTR = "openssl pkcs12 -export" +
+  " -in {2}" + File.separator + "{4} -inkey {2}" + File.separator +
+  "{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator +
+  "{5} -password pass:{1} -passin pass:{0} \n";
+  private static final String REVOKE_AGENT_CRT = "openssl ca " +
+  "-config {0}" + File.separator + "ca.config -keyfile {0}" +
+  File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " +
+  "-passin pass:{3} -cert {0}" + File.separator + "{5}";
+  private static final String SIGN_AGENT_CRT = "openssl ca -config " +
+  "{0}" + File.separator + "ca.config -in {0}" + File.separator +
+  "{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " +
+  "-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + 
"{5}";
+  private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
+  "rsa:1024 -nodes -keyout {0}" + File.separator +
+  "{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " +
+  "-config {3}" + File.separator + "ca.config ";
+  private String passphrase;
+  private String applicationName;
+
+
+  public void initialize(MapOperations compOperations) throws SliderException {
+String hostname = null;
+try {
+  hostname = InetAddress.getLocalHost().getCanonicalHostName();
+} catch (UnknownHostException e) {
+  hostname = "localhost";
+}
+this.initialize(compOperations, hostname, null, null);
+  }
+
+  /**
+* Verify that root certificate exists, generate it otherwise.
+*/
+  public void initialize(MapOperations compOperations,
+ String hostname, String containerId,
+ String appName) 

[01/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe) [Forced Update!]

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services b02252e21 -> 4945f85dc (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
new file mode 100644
index 000..b8bdc59
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.slider.api.proto";
+option java_outer_classname = "Messages";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package org.apache.slider.api;
+
+//import "Security.proto";
+
+/*
+  Look at SliderClusterProtocol.proto to see how to build this
+*/
+
+message RoleInstanceState {
+  required string name =1;
+  optional string role =2;
+  required uint32 state =   4;
+  required uint32 exitCode =5;
+  optional string command = 6;
+  optional string diagnostics = 7;
+  repeated string output =  8;
+  repeated string environment = 9;
+  required uint32 roleId = 10;
+  required bool released = 11;
+  required int64 createTime =  12;
+  required int64 startTime =   13;
+  required string host =   14;
+  required string hostURL =15;
+  optional string appVersion = 16;
+}
+
+/**
+ * stop the cluster
+ */
+message StopClusterRequestProto {
+  /**
+  message to include
+  */
+  required string message = 1;
+}
+
+/**
+ * stop the cluster
+ */
+message StopClusterResponseProto {
+}
+
+/**
+ * upgrade the containers
+ */
+message UpgradeContainersRequestProto {
+  /**
+  message to include
+  */
+  required string message = 1;
+  repeated string container =   2;
+  repeated string component =   3;
+}
+
+/**
+ * upgrade the containers
+ */
+message UpgradeContainersResponseProto {
+}
+
+/**
+ * flex the cluster
+ */
+message FlexClusterRequestProto {
+  required string clusterSpec = 1;
+}
+
+
+/**
+ * flex the cluster
+ */
+message FlexClusterResponseProto {
+  required bool response = 1;
+}
+
+
+/**
+ * void request
+ */
+message GetJSONClusterStatusRequestProto {
+}
+
+/**
+ * response
+ */
+message GetJSONClusterStatusResponseProto {
+  required string clusterSpec = 1;
+}
+
+/**
+ * list the nodes in a role
+ */
+message ListNodeUUIDsByRoleRequestProto {
+  required string role = 1;
+}
+
+/**
+ * list the nodes in a role
+ */
+message ListNodeUUIDsByRoleResponseProto {
+  repeated string uuid = 1 ;
+}
+
+/**
+ * get a node
+ */
+message GetNodeRequestProto {
+  required string uuid = 1;
+}
+
+
+/**
+ * response on a node
+ */
+message GetNodeResponseProto {
+   required RoleInstanceState clusterNode = 1 ;
+}
+
+/**
+ * list the nodes for the UUDs
+ */
+message GetClusterNodesRequestProto {
+  repeated string uuid = 1 ;
+}
+
+/**
+ * list the nodes in a role
+ */
+message GetClusterNodesResponseProto {
+  repeated RoleInstanceState clusterNode = 1 ;
+}
+
+/**
+ * Echo
+ */
+message EchoRequestProto {
+  required string text = 1;
+}
+
+/**
+ * Echo reply
+ */
+message EchoResponseProto {
+  required string text = 1;
+}
+
+
+/**
+ * Kill a container
+ */
+message KillContainerRequestProto {
+  required string id = 1;
+}
+
+/**
+ * Kill reply
+ */
+message KillContainerResponseProto {
+  required bool success = 1;
+}
+
+/**
+ * AM suicide
+ */
+message AMSuicideRequestProto {
+  required string text =  1;
+  required int32 signal = 2;
+  required int32 delay =  3;
+}
+
+/**
+ * AM suicide reply. For this to be returned implies
+ * a failure of the AM to kill itself
+ */
+message AMSuicideResponseProto {
+
+}
+
+
+/**
+ * Ask for the instance definition details
+ */
+message 

[04/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
new file mode 100644
index 000..9c14ca7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class HttpProbe extends Probe {
+  protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
+
+  private final URL url;
+  private final int timeout;
+  private final int min, max;
+
+
+  public HttpProbe(URL url, int timeout, int min, int max, Configuration conf) 
throws IOException {
+super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
+this.url = url;
+this.timeout = timeout;
+this.min = min;
+this.max = max;
+  }
+
+  public static HttpURLConnection getConnection(URL url, int timeout) throws 
IOException {
+HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+connection.setInstanceFollowRedirects(true);
+connection.setConnectTimeout(timeout);
+return connection;
+  }
+  
+  @Override
+  public ProbeStatus ping(boolean livePing) {
+ProbeStatus status = new ProbeStatus();
+HttpURLConnection connection = null;
+try {
+  if (log.isDebugEnabled()) {
+// LOG.debug("Fetching " + url + " with timeout " + timeout);
+  }
+  connection = getConnection(url, this.timeout);
+  int rc = connection.getResponseCode();
+  if (rc < min || rc > max) {
+String error = "Probe " + url + " error code: " + rc;
+log.info(error);
+status.fail(this,
+new IOException(error));
+  } else {
+status.succeed(this);
+  }
+} catch (IOException e) {
+  String error = "Probe " + url + " failed: " + e;
+  log.info(error, e);
+  status.fail(this,
+  new IOException(error, e));
+} finally {
+  if (connection != null) {
+connection.disconnect();
+  }
+}
+return status;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
new file mode 100644
index 000..a1ad44f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in 

[09/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
new file mode 100644
index 000..37e9a7f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Implementation of {@link StateAccessForProviders}, which means
+ * state access for providers, web UI and IPC/REST views.
+ */
+public class ProviderAppState implements StateAccessForProviders {
+
+
+  private final Map publishedConfigSets =
+  new ConcurrentHashMap<>(5);
+  private final PublishedExportsSet publishedExportsSets = new 
PublishedExportsSet();
+  private static final PatternValidator validator = new PatternValidator(
+  RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
+  private String applicationName;
+
+  private final AppState appState;
+
+  public ProviderAppState(String applicationName, AppState appState) {
+this.appState = appState;
+this.applicationName = applicationName;
+  }
+
+  public void setApplicationName(String applicationName) {
+this.applicationName = applicationName;
+  }
+
+  @Override
+  public String getApplicationName() {
+return applicationName;
+  }
+
+  @Override
+  public PublishedConfigSet getPublishedSliderConfigurations() {
+return getOrCreatePublishedConfigSet(RestPaths.SLIDER_CONFIGSET);
+  }
+
+  @Override
+  public PublishedExportsSet getPublishedExportsSet() {
+return publishedExportsSets;
+  }
+
+  @Override
+  public PublishedConfigSet getPublishedConfigSet(String name) {
+return publishedConfigSets.get(name);
+  }
+
+  @Override
+  public PublishedConfigSet getOrCreatePublishedConfigSet(String name) {
+PublishedConfigSet set = publishedConfigSets.get(name);
+if (set == null) {
+  validator.validate(name);
+  synchronized (publishedConfigSets) {
+// synchronized double check to ensure that there is never an 
overridden
+// config set created
+set = publishedConfigSets.get(name);
+if (set == null) {
+  set = new PublishedConfigSet();
+  publishedConfigSets.put(name, set);
+}
+  }
+}
+return set;
+  }
+
+  @Override
+  public List listConfigSets() {
+
+synchronized (publishedConfigSets) {
+  List sets = new 

[06/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
new file mode 100644
index 000..80b7a5e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegistrationResponse {
+
+  @JsonProperty("response")
+  private RegistrationStatus response;
+
+  /**
+   * exitstatus is a code of error which was rised on server side. exitstatus
+   * = 0 (OK - Default) exitstatus = 1 (Registration failed because different
+   * version of agent and server)
+   */
+  @JsonProperty("exitstatus")
+  private int exitstatus;
+
+  /** log - message, which will be printed to agents log */
+  @JsonProperty("log")
+  private String log;
+
+  /** tags - tags associated with the container */
+  @JsonProperty("tags")
+  private String tags;
+  
+  @JsonProperty("package")
+  private String pkg;
+
+  //Response id to start with, usually zero.
+  @JsonProperty("responseId")
+  private long responseId;
+
+  @JsonProperty("statusCommands")
+  private List statusCommands = null;
+
+  public RegistrationResponse() {
+  }
+
+  public RegistrationStatus getResponse() {
+return response;
+  }
+
+  public void setResponse(RegistrationStatus response) {
+this.response = response;
+  }
+
+  public int getExitstatus() {
+return exitstatus;
+  }
+
+  public void setExitstatus(int exitstatus) {
+this.exitstatus = exitstatus;
+  }
+
+  public RegistrationStatus getResponseStatus() {
+return response;
+  }
+
+  public void setResponseStatus(RegistrationStatus response) {
+this.response = response;
+  }
+
+  public List getStatusCommands() {
+return statusCommands;
+  }
+
+  public void setStatusCommands(List statusCommands) {
+this.statusCommands = statusCommands;
+  }
+
+  public long getResponseId() {
+return responseId;
+  }
+
+  public void setResponseId(long responseId) {
+this.responseId = responseId;
+  }
+
+  public String getTags() {
+return tags;
+  }
+
+  public void setTags(String tags) {
+this.tags = tags;
+  }
+
+  public String getLog() {
+return log;
+  }
+
+  public void setLog(String log) {
+this.log = log;
+  }
+
+  public String getPkg() {
+return pkg;
+  }
+
+  public void setPkg(String pkg) {
+this.pkg = pkg;
+  }
+
+  @Override
+  public String toString() {
+return "RegistrationResponse{" +
+   "response=" + response +
+   ", responseId=" + responseId +
+   ", statusCommands=" + statusCommands +
+   '}';
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
 

[05/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
new file mode 100644
index 000..a8e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ComponentResource {
+  private final Map props;
+  private String href;
+
+  public ComponentResource() {
+this(null, null, null, null);
+  }
+
+  public ComponentResource(String name,
+   Map props,
+   UriBuilder uriBuilder,
+   Map pathElems) {
+this.props = props;
+  }
+
+  public Map getProps() {
+return props;
+  }
+
+  public String getHref() {
+return href;
+  }
+
+  public void setHref(String href) {
+this.href = href;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
new file mode 100644
index 000..407bab6
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+

[02/63] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a379904d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
new file mode 100644
index 000..9e9e7ac
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -0,0 +1,598 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Execute a long-lived process.
+ *
+ * 
+ * Hadoop's {@link org.apache.hadoop.util.Shell} class assumes it is executing
+ * a short lived application; this class allows for the process to run for the
+ * life of the Java process that forked it.
+ * It is designed to be embedded inside a YARN service, though this is not
+ * the sole way that it can be used
+ * 
+ * Key Features:
+ * 
+ *   Output is streamed to the output logger provided.
+ *   the input stream is closed as soon as the process starts.
+ *   The most recent lines of output are saved to a linked list.
+ *   A synchronous callback, {@link LongLivedProcessLifecycleEvent},
+ *   is raised on the start and finish of a process.
+ * 
+ * 
+ */
+public class LongLivedProcess implements Runnable {
+  /**
+   * Limit on number of lines to retain in the "recent" line list:{@value}
+   */
+  public static final int RECENT_LINE_LOG_LIMIT = 64;
+
+  /**
+   * Const defining the time in millis between polling for new text.
+   */
+  private static final int STREAM_READER_SLEEP_TIME = 200;
+  
+  /**
+   * limit on the length of a stream before it triggers an automatic newline.
+   */
+  private static final int LINE_LENGTH = 256;
+  private final ProcessBuilder processBuilder;
+  private Process process;
+  private Integer exitCode = null;
+  private final String name;
+  private final ExecutorService processExecutor;
+  private final ExecutorService logExecutor;
+  
+  private ProcessStreamReader processStreamReader;
+  //list of recent lines, recorded for extraction into reports
+  private final List recentLines = new LinkedList<>();
+  private int recentLineLimit = RECENT_LINE_LOG_LIMIT;
+  private LongLivedProcessLifecycleEvent lifecycleCallback;
+  private final AtomicBoolean finalOutputProcessed = new AtomicBoolean(false);
+
+  /**
+   * Log supplied in the constructor for the spawned process -accessible
+   * to inner classes
+   */
+  private Logger processLog;
+  
+  /**
+   * Class log -accessible to inner classes
+   */
+  private static final Logger LOG = 
LoggerFactory.getLogger(LongLivedProcess.class);
+
+  /**
+   *  flag to indicate that the process is done
+   */
+  private final AtomicBoolean finished = new AtomicBoolean(false);
+
+  /**
+   * Create an instance
+   * @param name process name
+   * @param processLog log for output (or null)
+   * @param commands command list
+   */
+  public LongLivedProcess(String name,
+  Logger processLog,
+  List commands) {
+Preconditions.checkArgument(commands != 

hadoop git commit: HADOOP-13828. Implement getFileChecksum(path, length) for ViewFileSystem. Contributed by Manoj Govindassamy.

[wang]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 5d5614f84 -> a2b1ff025


HADOOP-13828. Implement getFileChecksum(path, length) for ViewFileSystem. 
Contributed by Manoj Govindassamy.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2b1ff02
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2b1ff02
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2b1ff02

Branch: refs/heads/trunk
Commit: a2b1ff0257bde26d1f64454e97bc1225294a30b9
Parents: 5d5614f
Author: Andrew Wang 
Authored: Mon Nov 28 11:54:43 2016 -0800
Committer: Andrew Wang 
Committed: Mon Nov 28 11:54:43 2016 -0800

--
 .../hadoop/fs/viewfs/ChRootedFileSystem.java|  6 
 .../apache/hadoop/fs/viewfs/ViewFileSystem.java |  9 ++
 .../fs/viewfs/TestViewFileSystemHdfs.java   | 29 
 3 files changed, 44 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b1ff02/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
index 9f61af6..272433f 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ChRootedFileSystem.java
@@ -222,6 +222,12 @@ class ChRootedFileSystem extends FilterFileSystem {
   }
 
   @Override
+  public FileChecksum getFileChecksum(final Path f, final long length)
+  throws IOException {
+return super.getFileChecksum(fullPath(f), length);
+  }
+
+  @Override
   public FileStatus getFileStatus(final Path f) 
   throws IOException {
 return super.getFileStatus(fullPath(f));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b1ff02/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
index 9061b2a..ed1bda2 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFileSystem.java
@@ -350,6 +350,15 @@ public class ViewFileSystem extends FileSystem {
 return res.targetFileSystem.getFileChecksum(res.remainingPath);
   }
 
+  @Override
+  public FileChecksum getFileChecksum(final Path f, final long length)
+  throws AccessControlException, FileNotFoundException,
+  IOException {
+InodeTree.ResolveResult res =
+fsState.resolve(getUriPath(f), true);
+return res.targetFileSystem.getFileChecksum(res.remainingPath, length);
+  }
+
   private static FileStatus fixFileStatus(FileStatus orig,
   Path qualified) throws IOException {
 // FileStatus#getPath is a fully qualified path relative to the root of

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2b1ff02/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
index 0e420d0..58b77f6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/viewfs/TestViewFileSystemHdfs.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileChecksum;
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystemTestHelper;
 import org.apache.hadoop.fs.FsConstants;
@@ -218,4 +220,31 @@ public class TestViewFileSystemHdfs extends 
ViewFileSystemBaseTest {
 DFSTestUtil.FsShellRun("-df /", 0, null, newConf);
 DFSTestUtil.FsShellRun("-df", 0, null, newConf);
   }
+
+  @Test
+  public void testFileChecksum() throws IOException {
+ViewFileSystem viewFs =