[Hadoop Wiki] Update of "Books" by Packt Publishing

[Apache Wiki]

Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change 
notification.

The "Books" page has been changed by Packt Publishing:
https://wiki.apache.org/hadoop/Books?action=diff=39=40

Comment:
URL change

  Unlock the power of your data with Hadoop 2.X ecosystem and its data 
warehousing techniques across large data sets.
  
  === Hadoop Explained (Free eBook Download) ===
- '''Name:''' 
[[https://www.packtpub.com/packt/free-ebook/hadoop-explained|Hadoop Explained]]
+ '''Name:''' 
[[https://www.packtpub.com/packt/free-ebook/hadoop-explained-2|Hadoop 
Explained]]
  
  '''Author:''' Aravind Shenoy
  

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by Sammi Chen and Andrew Wang

[drankye]

Repository: hadoop
Updated Branches:
  refs/heads/trunk ada876cd1 -> 1f14f6d03


HDFS-8411. Add bytes count metrics to datanode for ECWorker. Contributed by 
Sammi Chen and Andrew Wang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f14f6d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f14f6d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f14f6d0

Branch: refs/heads/trunk
Commit: 1f14f6d038aecad55a5398c6fa4137c9d2f44729
Parents: ada876c
Author: Kai Zheng 
Authored: Wed Dec 14 14:50:50 2016 +0800
Committer: Kai Zheng 
Committed: Wed Dec 14 14:50:50 2016 +0800

--
 .../erasurecode/StripedBlockReader.java |   1 +
 .../erasurecode/StripedBlockReconstructor.java  |   6 +-
 .../erasurecode/StripedBlockWriter.java |   1 +
 .../datanode/erasurecode/StripedReader.java |   4 +
 .../erasurecode/StripedReconstructor.java   |  21 +++
 .../datanode/erasurecode/StripedWriter.java |   4 +
 .../datanode/metrics/DataNodeMetrics.java   |  18 ++-
 .../apache/hadoop/hdfs/StripedFileTestUtil.java |  24 +++
 .../TestDataNodeErasureCodingMetrics.java   | 149 +--
 9 files changed, 147 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
index a27de9b..0f7c5c7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReader.java
@@ -187,6 +187,7 @@ class StripedBlockReader {
 break;
   }
   n += nread;
+  stripedReader.getReconstructor().incrBytesRead(nread);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
index a8e9d30..5554d68 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockReconstructor.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 
 import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics;
 
 /**
  * StripedBlockReconstructor reconstruct one or more missed striped block in
@@ -66,7 +67,10 @@ class StripedBlockReconstructor extends StripedReconstructor
   getDatanode().getMetrics().incrECFailedReconstructionTasks();
 } finally {
   getDatanode().decrementXmitsInProgress();
-  getDatanode().getMetrics().incrECReconstructionTasks();
+  final DataNodeMetrics metrics = getDatanode().getMetrics();
+  metrics.incrECReconstructionTasks();
+  metrics.incrECReconstructionBytesRead(getBytesRead());
+  metrics.incrECReconstructionBytesWritten(getBytesWritten());
   getStripedReader().close();
   stripedWriter.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f14f6d0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
index 592be45..d999202 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedBlockWriter.java
@@ -196,6 +196,7 @@ class StripedBlockWriter {
   

hadoop git commit: Revert YARN-4126. RM should not issue delegation tokens in unsecure mode.

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk fbdbbd57c -> ada876cd1


Revert YARN-4126. RM should not issue delegation tokens in unsecure mode.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ada876cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ada876cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ada876cd

Branch: refs/heads/trunk
Commit: ada876cd1d22b61f237603cf339bbed65285dab8
Parents: fbdbbd5
Author: Jian He 
Authored: Tue Dec 13 20:49:54 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 20:49:54 2016 -0800

--
 .../apache/hadoop/yarn/server/resourcemanager/ClientRMService.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada876cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 1bc40c4..3dc7e38 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1238,7 +1238,7 @@ public class ClientRMService extends AbstractService 
implements
   .contains(UserGroupInformation.getCurrentUser()
   .getRealAuthenticationMethod());
 } else {
-  return false;
+  return true;
 }
   }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-10684. WebHDFS DataNode calls fail without parameter createparent. Contributed by John Zhuge.

[wang]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0af67aa20 -> 6b795c34d


HDFS-10684. WebHDFS DataNode calls fail without parameter createparent. 
Contributed by John Zhuge.

(cherry picked from commit fbdbbd57cdc3d8c778fca9266a7cadf298c8ff6c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b795c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b795c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b795c34

Branch: refs/heads/branch-2
Commit: 6b795c34d0c534a8d5214c64036ba73708d58fd0
Parents: 0af67aa
Author: Andrew Wang 
Authored: Tue Dec 13 18:01:31 2016 -0800
Committer: Andrew Wang 
Committed: Tue Dec 13 18:01:35 2016 -0800

--
 .../hdfs/web/resources/CreateParentParam.java   |  2 +-
 .../hdfs/web/resources/OverwriteParam.java  |  2 +-
 .../hdfs/web/TestWebHdfsFileSystemContract.java | 40 
 3 files changed, 42 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b795c34/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
index eaa5e8d..029efa0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
@@ -39,7 +39,7 @@ public class CreateParentParam extends BooleanParam {
* @param str a string representation of the parameter value.
*/
   public CreateParentParam(final String str) {
-this(DOMAIN.parse(str));
+this(DOMAIN.parse(str == null ? DEFAULT : str));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b795c34/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
index 9610b93..d7f5fb0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
@@ -39,7 +39,7 @@ public class OverwriteParam extends BooleanParam {
* @param str a string representation of the parameter value.
*/
   public OverwriteParam(final String str) {
-this(DOMAIN.parse(str));
+super(DOMAIN, DOMAIN.parse(str == null ? DEFAULT : str));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b795c34/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
index 45b363a..925edcf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.text.MessageFormat;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Random;
@@ -49,6 +50,7 @@ import 
org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -533,6 +535,44 @@ public class TestWebHdfsFileSystemContract extends 
FileSystemContractBaseTest {
 }
   }
 
+  public void testDatanodeCreateMissingParameter() throws IOException {
+final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) fs;
+final Path testDir = new Path(MessageFormat.format("/test/{0}/{1}",
+TestWebHdfsFileSystemContract.class,
+GenericTestUtils.getMethodName()));
+

hadoop git commit: HDFS-10684. WebHDFS DataNode calls fail without parameter createparent. Contributed by John Zhuge.

[wang]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 120f3a0ff -> 158292fd7


HDFS-10684. WebHDFS DataNode calls fail without parameter createparent. 
Contributed by John Zhuge.

(cherry picked from commit fbdbbd57cdc3d8c778fca9266a7cadf298c8ff6c)
(cherry picked from commit 6b795c34d0c534a8d5214c64036ba73708d58fd0)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/158292fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/158292fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/158292fd

Branch: refs/heads/branch-2.8
Commit: 158292fd7e6ca1769f8598dd8b4153a9f3b043d9
Parents: 120f3a0
Author: Andrew Wang 
Authored: Tue Dec 13 18:01:31 2016 -0800
Committer: Andrew Wang 
Committed: Tue Dec 13 18:01:40 2016 -0800

--
 .../hdfs/web/resources/CreateParentParam.java   |  2 +-
 .../hdfs/web/resources/OverwriteParam.java  |  2 +-
 .../hdfs/web/TestWebHdfsFileSystemContract.java | 40 
 3 files changed, 42 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/158292fd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
index eaa5e8d..029efa0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
@@ -39,7 +39,7 @@ public class CreateParentParam extends BooleanParam {
* @param str a string representation of the parameter value.
*/
   public CreateParentParam(final String str) {
-this(DOMAIN.parse(str));
+this(DOMAIN.parse(str == null ? DEFAULT : str));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/158292fd/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
index 9610b93..d7f5fb0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
@@ -39,7 +39,7 @@ public class OverwriteParam extends BooleanParam {
* @param str a string representation of the parameter value.
*/
   public OverwriteParam(final String str) {
-this(DOMAIN.parse(str));
+super(DOMAIN, DOMAIN.parse(str == null ? DEFAULT : str));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/158292fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
index 45b363a..925edcf 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.text.MessageFormat;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Random;
@@ -49,6 +50,7 @@ import 
org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -533,6 +535,44 @@ public class TestWebHdfsFileSystemContract extends 
FileSystemContractBaseTest {
 }
   }
 
+  public void testDatanodeCreateMissingParameter() throws IOException {
+final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) fs;
+final Path testDir = new Path(MessageFormat.format("/test/{0}/{1}",
+

hadoop git commit: HDFS-10684. WebHDFS DataNode calls fail without parameter createparent. Contributed by John Zhuge.

[wang]

Repository: hadoop
Updated Branches:
  refs/heads/trunk e24a923db -> fbdbbd57c


HDFS-10684. WebHDFS DataNode calls fail without parameter createparent. 
Contributed by John Zhuge.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbdbbd57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbdbbd57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbdbbd57

Branch: refs/heads/trunk
Commit: fbdbbd57cdc3d8c778fca9266a7cadf298c8ff6c
Parents: e24a923
Author: Andrew Wang 
Authored: Tue Dec 13 18:01:31 2016 -0800
Committer: Andrew Wang 
Committed: Tue Dec 13 18:01:31 2016 -0800

--
 .../hdfs/web/resources/CreateParentParam.java   |  2 +-
 .../hdfs/web/resources/OverwriteParam.java  |  2 +-
 .../hdfs/web/TestWebHdfsFileSystemContract.java | 40 
 3 files changed, 42 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbdbbd57/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
index eaa5e8d..029efa0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/CreateParentParam.java
@@ -39,7 +39,7 @@ public class CreateParentParam extends BooleanParam {
* @param str a string representation of the parameter value.
*/
   public CreateParentParam(final String str) {
-this(DOMAIN.parse(str));
+this(DOMAIN.parse(str == null ? DEFAULT : str));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbdbbd57/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
index 9610b93..d7f5fb0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/OverwriteParam.java
@@ -39,7 +39,7 @@ public class OverwriteParam extends BooleanParam {
* @param str a string representation of the parameter value.
*/
   public OverwriteParam(final String str) {
-this(DOMAIN.parse(str));
+super(DOMAIN, DOMAIN.parse(str == null ? DEFAULT : str));
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbdbbd57/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
index a68b1ac..4854471 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
@@ -25,6 +25,7 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.text.MessageFormat;
 import java.util.Arrays;
 import java.util.Map;
 import java.util.Random;
@@ -48,6 +49,7 @@ import 
org.apache.hadoop.hdfs.web.resources.NamenodeAddressParam;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -531,6 +533,44 @@ public class TestWebHdfsFileSystemContract extends 
FileSystemContractBaseTest {
 }
   }
 
+  public void testDatanodeCreateMissingParameter() throws IOException {
+final WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) fs;
+final Path testDir = new Path(MessageFormat.format("/test/{0}/{1}",
+TestWebHdfsFileSystemContract.class,
+GenericTestUtils.getMethodName()));
+assertTrue(webhdfs.mkdirs(testDir));
+
+for (String dnCreateParam : new 

hadoop git commit: HDFS-11164: Mover should avoid unnecessary retries if the block is pinned. Contributed by Rakesh R

[umamahesh]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 9947aeb60 -> e24a923db


HDFS-11164: Mover should avoid unnecessary retries if the block is pinned. 
Contributed by Rakesh R


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e24a923d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e24a923d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e24a923d

Branch: refs/heads/trunk
Commit: e24a923db50879f7dbe5d2afac0e6757089fb07d
Parents: 9947aeb
Author: Uma Maheswara Rao G 
Authored: Tue Dec 13 17:09:58 2016 -0800
Committer: Uma Maheswara Rao G 
Committed: Tue Dec 13 17:09:58 2016 -0800

--
 .../datatransfer/BlockPinningException.java |  33 
 .../datatransfer/DataTransferProtoUtil.java |  17 +-
 .../src/main/proto/datatransfer.proto   |   1 +
 .../hadoop/hdfs/server/balancer/Dispatcher.java |  62 ++-
 .../hdfs/server/datanode/DataXceiver.java   |   8 +-
 .../apache/hadoop/hdfs/server/mover/Mover.java  |  26 ++-
 .../hdfs/server/datanode/DataNodeTestUtils.java |  28 
 .../server/datanode/TestBlockReplacement.java   |  70 +++-
 .../hadoop/hdfs/server/mover/TestMover.java | 163 ++-
 9 files changed, 395 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24a923d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlockPinningException.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlockPinningException.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlockPinningException.java
new file mode 100644
index 000..c2f12f9
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/BlockPinningException.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol.datatransfer;
+
+import java.io.IOException;
+
+/**
+ * Indicates a failure due to block pinning.
+ */
+public class BlockPinningException extends IOException {
+
+  // Required by {@link java.io.Serializable}.
+  private static final long serialVersionUID = 1L;
+
+  public BlockPinningException(String errMsg) {
+super(errMsg);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e24a923d/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
index 6801149..287928c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/DataTransferProtoUtil.java
@@ -24,11 +24,11 @@ import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BaseHeaderProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
-import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ChecksumProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ClientOperationHeaderProto;
-import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.OpWriteBlockProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.DataTransferTraceInfoProto;
+import 

hadoop git commit: HADOOP-13709. Ability to clean up subprocesses spawned by Shell when the process exits. Contributed by Eric Badger (cherry picked from commit 9947aeb60c3dd075544866fd6e4dab0ad8b4afa

[jlowe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d7d9006af -> 0af67aa20


HADOOP-13709. Ability to clean up subprocesses spawned by Shell when the 
process exits. Contributed by Eric Badger
(cherry picked from commit 9947aeb60c3dd075544866fd6e4dab0ad8b4afa2)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0af67aa2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0af67aa2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0af67aa2

Branch: refs/heads/branch-2
Commit: 0af67aa20014b355d43d74d52650039cf95c0a54
Parents: d7d9006
Author: Jason Lowe 
Authored: Tue Dec 13 22:55:09 2016 +
Committer: Jason Lowe 
Committed: Tue Dec 13 22:57:41 2016 +

--
 .../main/java/org/apache/hadoop/util/Shell.java | 24 +++
 .../java/org/apache/hadoop/util/TestShell.java  | 68 
 2 files changed, 92 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af67aa2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index b6302ea..55f92d6 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -26,9 +26,11 @@ import java.io.InputStream;
 import java.io.InterruptedIOException;
 import java.nio.charset.Charset;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.WeakHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -48,6 +50,8 @@ import org.slf4j.LoggerFactory;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public abstract class Shell {
+  private static final Map  CHILD_PROCESSES =
+  Collections.synchronizedMap(new WeakHashMap());
   public static final Logger LOG = LoggerFactory.getLogger(Shell.class);
 
   /**
@@ -920,6 +924,7 @@ public abstract class Shell {
 } else {
   process = builder.start();
 }
+CHILD_PROCESSES.put(process, null);
 
 if (timeOutInterval > 0) {
   timeOutTimer = new Timer("Shell command timeout");
@@ -1016,6 +1021,7 @@ public abstract class Shell {
 LOG.warn("Error while closing the error stream", ioe);
   }
   process.destroy();
+  CHILD_PROCESSES.remove(process);
   lastTime = Time.monotonicNow();
 }
   }
@@ -1314,4 +1320,22 @@ public abstract class Shell {
   }
 }
   }
+
+  /**
+   * Static method to destroy all running Shell processes
+   * Iterates through a list of all currently running Shell
+   * processes and destroys them one by one. This method is thread safe and
+   * is intended to be used in a shutdown hook.
+   */
+  public static void destroyAllProcesses() {
+synchronized (CHILD_PROCESSES) {
+  for (Process key : CHILD_PROCESSES.keySet()) {
+Process process = key;
+if (key != null) {
+  process.destroy();
+}
+  }
+  CHILD_PROCESSES.clear();
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af67aa2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index 3ef6fb1..0845992 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.util;
 
+import com.google.common.base.Supplier;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
 import org.junit.Assert;
@@ -476,4 +477,71 @@ public class TestShell extends Assert {
 assertEquals("'foo'\\''bar'", Shell.bashQuote("foo'bar"));
 assertEquals("''\\''foo'\\''bar'\\'''", Shell.bashQuote("'foo'bar'"));
   }
+
+  @Test(timeout=12)
+  public void testShellKillAllProcesses() throws Throwable {
+Assume.assumeFalse(WINDOWS);
+StringBuffer sleepCommand = new StringBuffer();
+sleepCommand.append("sleep 200");
+String[] shellCmd = {"bash", "-c", sleepCommand.toString()};
+final ShellCommandExecutor shexc1 = new 

hadoop git commit: HADOOP-13709. Ability to clean up subprocesses spawned by Shell when the process exits. Contributed by Eric Badger

[jlowe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk ef34bf2bb -> 9947aeb60


HADOOP-13709. Ability to clean up subprocesses spawned by Shell when the 
process exits. Contributed by Eric Badger


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9947aeb6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9947aeb6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9947aeb6

Branch: refs/heads/trunk
Commit: 9947aeb60c3dd075544866fd6e4dab0ad8b4afa2
Parents: ef34bf2
Author: Jason Lowe 
Authored: Tue Dec 13 22:55:09 2016 +
Committer: Jason Lowe 
Committed: Tue Dec 13 22:55:09 2016 +

--
 .../main/java/org/apache/hadoop/util/Shell.java | 24 +++
 .../java/org/apache/hadoop/util/TestShell.java  | 68 
 2 files changed, 92 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9947aeb6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 5fc9869..83877b7 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -26,9 +26,11 @@ import java.io.InputStream;
 import java.io.InterruptedIOException;
 import java.nio.charset.Charset;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Map;
 import java.util.Timer;
 import java.util.TimerTask;
+import java.util.WeakHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -48,6 +50,8 @@ import org.slf4j.LoggerFactory;
 @InterfaceAudience.Public
 @InterfaceStability.Evolving
 public abstract class Shell {
+  private static final Map  CHILD_PROCESSES =
+  Collections.synchronizedMap(new WeakHashMap());
   public static final Logger LOG = LoggerFactory.getLogger(Shell.class);
 
   /**
@@ -916,6 +920,7 @@ public abstract class Shell {
 } else {
   process = builder.start();
 }
+CHILD_PROCESSES.put(process, null);
 
 if (timeOutInterval > 0) {
   timeOutTimer = new Timer("Shell command timeout");
@@ -1012,6 +1017,7 @@ public abstract class Shell {
 LOG.warn("Error while closing the error stream", ioe);
   }
   process.destroy();
+  CHILD_PROCESSES.remove(process);
   lastTime = Time.monotonicNow();
 }
   }
@@ -1310,4 +1316,22 @@ public abstract class Shell {
   }
 }
   }
+
+  /**
+   * Static method to destroy all running Shell processes
+   * Iterates through a list of all currently running Shell
+   * processes and destroys them one by one. This method is thread safe and
+   * is intended to be used in a shutdown hook.
+   */
+  public static void destroyAllProcesses() {
+synchronized (CHILD_PROCESSES) {
+  for (Process key : CHILD_PROCESSES.keySet()) {
+Process process = key;
+if (key != null) {
+  process.destroy();
+}
+  }
+  CHILD_PROCESSES.clear();
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9947aeb6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index 67903f7..5cc011b 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.util;
 
+import com.google.common.base.Supplier;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
 import org.junit.Assert;
@@ -471,4 +472,71 @@ public class TestShell extends Assert {
 assertEquals("'foo'\\''bar'", Shell.bashQuote("foo'bar"));
 assertEquals("''\\''foo'\\''bar'\\'''", Shell.bashQuote("'foo'bar'"));
   }
+
+  @Test(timeout=12)
+  public void testShellKillAllProcesses() throws Throwable {
+Assume.assumeFalse(WINDOWS);
+StringBuffer sleepCommand = new StringBuffer();
+sleepCommand.append("sleep 200");
+String[] shellCmd = {"bash", "-c", sleepCommand.toString()};
+final ShellCommandExecutor shexc1 = new ShellCommandExecutor(shellCmd);
+final ShellCommandExecutor shexc2 = new 

[53/74] [abbrv] hadoop git commit: YARN-5769. Integrate update app lifetime using feature implemented in YARN-5611. Contributed by Jian He

[jianhe]

YARN-5769. Integrate update app lifetime using feature implemented in 
YARN-5611. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c7c57c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c7c57c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c7c57c0

Branch: refs/heads/yarn-native-services
Commit: 0c7c57c0539487dfcfbb47d4661811b9a22857d5
Parents: 40529b0
Author: Gour Saha 
Authored: Wed Nov 30 23:24:45 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../api/impl/ApplicationApiService.java | 25 +--
 .../org/apache/slider/client/SliderClient.java  | 33 +---
 .../AbstractClusterBuildingActionArgs.java  |  3 +-
 3 files changed, 53 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c7c57c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 6db69ac..c4f5d43 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -77,6 +77,7 @@ import org.apache.slider.common.params.ActionFreezeArgs;
 import org.apache.slider.common.params.ActionListArgs;
 import org.apache.slider.common.params.ActionRegistryArgs;
 import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionUpdateArgs;
 import org.apache.slider.common.params.ComponentArgsDelegate;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.common.tools.SliderVersionInfo;
@@ -1398,14 +1399,34 @@ public class ApplicationApiService implements 
ApplicationApi {
 }
 
 // If new lifetime value specified then update it
-if (updateAppData.getLifetime() != null) {
-  // TODO: Once YARN-3813 and YARN-4205 are available
+if (updateAppData.getLifetime() != null
+&& updateAppData.getLifetime() > 0) {
+  try {
+updateAppLifetime(appName, updateAppData.getLifetime());
+  } catch (Exception e) {
+logger.error("Failed to update application (" + appName + ") lifetime 
("
++ updateAppData.getLifetime() + ")", e);
+return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+  }
 }
 
 // If nothing happens consider it a no-op
 return Response.status(Status.NO_CONTENT).build();
   }
 
+  private Void updateAppLifetime(String appName, long lifetime)
+  throws InterruptedException, YarnException, IOException {
+return invokeSliderClientRunnable(new SliderClientContextRunnable() {
+  @Override public Void run(SliderClient sliderClient)
+  throws YarnException, IOException, InterruptedException {
+ActionUpdateArgs args = new ActionUpdateArgs();
+args.lifetime = lifetime;
+sliderClient.actionUpdate(appName, args);
+return null;
+  }
+});
+  }
+
   // create default component and initialize with app level global values
   private List getDefaultComponentAsList(Application app) {
 List comps = getDefaultComponentAsList();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c7c57c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3f2df0a..12f7870 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 

[56/74] [abbrv] hadoop git commit: YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha

[jianhe]

YARN-5883 Avoid or eliminate expensive YARN get all applications call. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdea5e1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdea5e1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdea5e1d

Branch: refs/heads/yarn-native-services
Commit: fdea5e1d2b1d1967540570486fffb081ee032ea0
Parents: 038561c
Author: Billie Rinaldi 
Authored: Tue Nov 29 13:14:45 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../org/apache/slider/client/SliderClient.java  |  6 +-
 .../slider/client/SliderYarnClientImpl.java | 94 +---
 .../apache/slider/common/tools/SliderUtils.java | 37 +++-
 .../slider/core/registry/YarnAppListClient.java | 21 -
 .../providers/AbstractClientProvider.java   | 28 +-
 .../providers/docker/DockerClientProvider.java  |  8 ++
 .../servicemonitor/YarnApplicationProbe.java| 12 +--
 7 files changed, 160 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ea10ed0..3f2df0a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -198,6 +198,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -2083,7 +2084,7 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 
 // add the tags if available
 Set applicationTags = provider.getApplicationTags(sliderFileSystem,
-appOperations);
+appOperations, clustername);
 
 Credentials credentials = null;
 if (clusterSecure) {
@@ -3031,9 +3032,10 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
 } else {
   // scan for instance in single --state state
-  List userInstances = 
yarnClient.listDeployedInstances("");
   state = state.toUpperCase(Locale.ENGLISH);
   YarnApplicationState desiredState = extractYarnApplicationState(state);
+  List userInstances = yarnClient
+  .listDeployedInstances("", EnumSet.of(desiredState), name);
   ApplicationReport foundInstance =
   yarnClient.findAppInInstanceList(userInstances, name, desiredState);
   if (foundInstance != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdea5e1d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index d471cdb..258ef31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -53,6 +53,8 @@ import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
   public List listDeployedInstances(String user)
 throws YarnException, 

[58/74] [abbrv] hadoop git commit: YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

[jianhe]

YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c22be358
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c22be358
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c22be358

Branch: refs/heads/yarn-native-services
Commit: c22be358a5e3470b6c0a705082e1505c67a4463f
Parents: be43905
Author: Billie Rinaldi 
Authored: Tue Dec 13 10:16:09 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../slider/api/SliderClusterProtocol.java   |   3 -
 .../slider/api/proto/RestTypeMarshalling.java   |  36 --
 .../org/apache/slider/client/SliderClient.java  |  80 ---
 .../client/ipc/SliderClusterOperations.java |  28 +-
 .../slider/common/params/ActionClientArgs.java  |  31 +-
 .../apache/slider/providers/ProviderUtils.java  | 141 -
 .../providers/docker/DockerProviderService.java |   5 -
 .../server/appmaster/SliderAppMaster.java   |  18 +-
 .../rpc/SliderClusterProtocolPBImpl.java|  12 -
 .../rpc/SliderClusterProtocolProxy.java |  18 +-
 .../server/appmaster/rpc/SliderIPCService.java  |  54 +-
 .../slider/server/appmaster/web/WebAppApi.java  |  23 -
 .../server/appmaster/web/WebAppApiImpl.java |  37 +-
 .../AbstractSecurityStoreGenerator.java |  98 
 .../services/security/CertificateManager.java   | 495 -
 .../services/security/KeystoreGenerator.java|  64 ---
 .../server/services/security/SecurityStore.java |  66 ---
 .../security/SecurityStoreGenerator.java|  40 --
 .../server/services/security/SecurityUtils.java | 256 -
 .../services/security/SignCertResponse.java |  67 ---
 .../server/services/security/SignMessage.java   |  54 --
 .../services/security/StoresGenerator.java  |  68 ---
 .../services/security/TruststoreGenerator.java  |  62 ---
 .../src/main/proto/SliderClusterProtocol.proto  |   6 -
 .../security/TestCertificateManager.java| 540 ---
 .../TestMultiThreadedStoreGeneration.java   | 156 --
 26 files changed, 14 insertions(+), 2444 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22be358/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 33fce22..893e706 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -173,7 +173,4 @@ public interface SliderClusterProtocol extends 
VersionedProtocol {
   Messages.WrappedJsonProto 
getModelResolvedResources(Messages.EmptyPayloadProto request) throws 
IOException;
 
   Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto 
request) throws IOException;
-
-  Messages.GetCertificateStoreResponseProto 
getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
-  throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22be358/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 17fd965..ec35028 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -18,8 +18,6 @@
 
 package org.apache.slider.api.proto;
 
-import com.google.protobuf.ByteString;
-import 

[50/74] [abbrv] hadoop git commit: YARN-5796. Convert enums values in service code to upper case and special handling of an error. Contributed by Gour Saha

[jianhe]

YARN-5796. Convert enums values in service code to upper case and special 
handling of an error. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a3f1a4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a3f1a4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a3f1a4a

Branch: refs/heads/yarn-native-services
Commit: 4a3f1a4a39022e42e8b4632ecc2a92f43fd375b5
Parents: f965dd1
Author: Jian He 
Authored: Tue Nov 1 11:00:11 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../yarn/services/api/impl/ApplicationApiService.java |  4 
 .../apache/hadoop/yarn/services/resource/Artifact.java|  2 +-
 .../apache/hadoop/yarn/services/resource/ConfigFile.java  |  4 ++--
 .../hadoop/yarn/services/resource/ReadinessCheck.java |  2 +-
 .../hadoop/yarn/services/utils/RestApiErrorMessages.java  | 10 +++---
 5 files changed, 15 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a3f1a4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index cf43ac2..37bd134 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -172,6 +172,10 @@ public class ApplicationApiService implements 
ApplicationApi {
 applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE);
 return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
 .build();
+  } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS);
+return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+.build();
   } else {
 applicationStatus.setDiagnostics(se.getMessage());
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a3f1a4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
index 9ac2bc7..87fcf89 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -40,7 +40,7 @@ public class Artifact implements Serializable {
   private String id = null;
 
   public enum TypeEnum {
-DOCKER("docker"), TARBALL("tarball"), APPLICATION("application");
+DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
 
 private String value;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a3f1a4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
index 3ced153..01d976f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
+++ 

[59/74] [abbrv] hadoop git commit: YARN-5944. Native services AM should remain up if RM is down. Contributed by Billie Rinaldi

[jianhe]

YARN-5944. Native services AM should remain up if RM is down. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1abf9171
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1abf9171
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1abf9171

Branch: refs/heads/yarn-native-services
Commit: 1abf9171fc3ae45c36822301254ba4ff39d0135b
Parents: 0c7c57c
Author: Gour Saha 
Authored: Thu Dec 1 00:30:01 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../org/apache/slider/server/appmaster/SliderAppMaster.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1abf9171/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 34b6a7d..8c39343 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -705,6 +705,11 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 synchronized (appState) {
   int heartbeatInterval = HEARTBEAT_INTERVAL;
 
+  // configure AM to wait forever for RM
+  
getConfig().setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+  -1);
+  getConfig().unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
   // add the RM client -this brings the callbacks in
   asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, 
this);
   addService(asyncRMClient);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[48/74] [abbrv] hadoop git commit: YARN-5812. Exception during GET call - "Failed to retrieve application: null". Contributed by Gour Saha

[jianhe]

YARN-5812. Exception during GET call - "Failed to retrieve application: null". 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a9d70cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a9d70cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a9d70cf

Branch: refs/heads/yarn-native-services
Commit: 8a9d70cfccbf35b05a3c1d6dbcf76ffc1112582d
Parents: eb8bd39
Author: Jian He 
Authored: Wed Nov 2 15:55:48 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../api/impl/ApplicationApiService.java | 36 ++--
 1 file changed, 26 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a9d70cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 37bd134..6db69ac 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,7 +50,6 @@ import javax.ws.rs.core.Response.Status;
 
 import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -905,6 +904,10 @@ public class ApplicationApiService implements 
ApplicationApi {
 
 // state
 String appState = jsonGetAsString(appStatus, "state");
+if (appState == null) {
+  // consider that app is still in ACCEPTED state
+  appState = String.valueOf(StateValues.STATE_INCOMPLETE);
+}
 switch (Integer.parseInt(appState)) {
   case StateValues.STATE_LIVE:
 app.setState(ApplicationState.STARTED);
@@ -1069,6 +1072,9 @@ public class ApplicationApiService implements 
ApplicationApi {
 String status = null;
 try {
   status = sliderClient.actionStatus(appName);
+} catch (BadClusterStateException e) {
+  logger.warn("Application not running yet", e);
+  return EMPTY_JSON_OBJECT;
 } catch (Exception e) {
   logger.error("Exception calling slider.actionStatus", e);
   return EMPTY_JSON_OBJECT;
@@ -1097,7 +1103,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 try {
   registry = sliderClient.actionRegistryGetConfig(registryArgs)
 .asJson();
-} catch (FileNotFoundException | PathNotFoundException e) {
+} catch (FileNotFoundException | NotFoundException e) {
   // ignore and return empty object
   return EMPTY_JSON_OBJECT;
 } catch (Exception e) {
@@ -1192,23 +1198,33 @@ public class ApplicationApiService implements 
ApplicationApi {
 // little longer for it to stop from YARN point of view. Slider destroy
 // fails if the application is not completely stopped. Hence the need to
 // call destroy in a controlled loop few times (only if exit code is
-// EXIT_APPLICATION_IN_USE), before giving up.
+// EXIT_APPLICATION_IN_USE or EXIT_INSTANCE_EXISTS), before giving up.
 boolean keepTrying = true;
-int maxDeleteAttempt = 5;
-int deleteAttempt = 0;
-while (keepTrying && deleteAttempt < maxDeleteAttempt) {
+int maxDeleteAttempts = 5;
+int deleteAttempts = 0;
+int sleepIntervalInMillis = 500;
+while (keepTrying && deleteAttempts < maxDeleteAttempts) {
   try {
 destroySliderApplication(appName);
 keepTrying = false;
   } catch (SliderException e) {
-logger.error("Delete application threw exception", e);
-if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE) {
-  deleteAttempt++;
+if (e.getExitCode() == SliderExitCodes.EXIT_APPLICATION_IN_USE
+|| e.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+  deleteAttempts++;
+  // If we used up all the allowed 

[73/74] [abbrv] hadoop git commit: YARN-5689. Update native services REST API to use agentless docker provider. Contributed by Billie Rinaldi & Gour Saha

[jianhe]

YARN-5689. Update native services REST API to use agentless docker provider. 
Contributed by Billie Rinaldi & Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/713018db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/713018db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/713018db

Branch: refs/heads/yarn-native-services
Commit: 713018dba9ac7e7c81e91506c421e68f11f7173c
Parents: f89aa15
Author: Jian He 
Authored: Thu Oct 13 11:34:58 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../api/impl/ApplicationApiService.java | 251 ++-
 .../yarn/services/utils/RestApiConstants.java   |   3 -
 .../api/impl/TestApplicationApiService.java |   6 +-
 3 files changed, 79 insertions(+), 181 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/713018db/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 9645696..0a62629 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,7 +50,6 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -68,10 +67,12 @@ import 
org.apache.hadoop.yarn.services.resource.Configuration;
 import org.apache.hadoop.yarn.services.resource.Container;
 import org.apache.hadoop.yarn.services.resource.ContainerState;
 import org.apache.hadoop.yarn.services.resource.Resource;
+import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.StateValues;
 import org.apache.slider.client.SliderClient;
 import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.params.ActionCreateArgs;
 import org.apache.slider.common.params.ActionFlexArgs;
 import org.apache.slider.common.params.ActionFreezeArgs;
@@ -88,12 +89,11 @@ import org.apache.slider.core.exceptions.NotFoundException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.providers.docker.DockerKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
@@ -211,7 +211,8 @@ public class ApplicationApiService implements 
ApplicationApi {
   application.setConfiguration(new Configuration());
 }
 addPropertyToConfiguration(application.getConfiguration(),
-PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+SliderKeys.COMPONENT_TYPE_KEY,
+SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
   }
   // resource
   validateApplicationResource(application.getResource(), null, application
@@ -249,7 +250,8 @@ public class ApplicationApiService implements 
ApplicationApi {
 comp.setConfiguration(new Configuration());
   }
   addPropertyToConfiguration(comp.getConfiguration(),
-  PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+  SliderKeys.COMPONENT_TYPE_KEY,
+  SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
   compNameArtifactIdMap.put(comp.getName(), 
comp.getArtifact().getId());
   comp.setName(comp.getArtifact().getId());
 }
@@ -339,9 +341,9 @@ public class ApplicationApiService implements 
ApplicationApi {
 final ActionCreateArgs createArgs = new ActionCreateArgs();
 addAppConfOptions(createArgs, application, compNameArtifactIdMap);
 

[65/74] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index a0fe310..bd4d2bf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.state.RoleStatus;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
 import 
org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
 import org.apache.slider.server.services.security.CertificateManager;
 import org.slf4j.Logger;
@@ -97,11 +96,6 @@ public class WebAppApiImpl implements WebAppApi {
   }
 
   @Override
-  public AgentRestOperations getAgentRestOperations() {
-return provider.getAgentRestOperations();
-  }
-
-  @Override
   public RegistryOperations getRegistryOperations() {
 return registryOperations;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
index 03bf703..aed87d8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.rest;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
 import 
org.apache.slider.server.appmaster.web.rest.application.ApplicationResource;
 import 
org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
 import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
deleted file mode 100644
index 17cd8f2..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless 

[74/74] [abbrv] hadoop git commit: YARN-5778. Add .keep file for yarn native services AM web app. Contributed by Billie Rinaldi

[jianhe]

YARN-5778. Add .keep file for yarn native services AM web app. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53e885dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53e885dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53e885dd

Branch: refs/heads/yarn-native-services
Commit: 53e885dd04ace1cb00274147419456fb223d7eca
Parents: 813e132
Author: Gour Saha 
Authored: Tue Oct 25 10:06:16 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../src/main/resources/webapps/slideram/.keep| 0
 1 file changed, 0 insertions(+), 0 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/53e885dd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/webapps/slideram/.keep
new file mode 100644
index 000..e69de29


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[63/74] [abbrv] hadoop git commit: YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He

[jianhe]

YARN-5735. Make the service REST API use the app timeout feature YARN-4205. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf6a2e43
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf6a2e43
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf6a2e43

Branch: refs/heads/yarn-native-services
Commit: cf6a2e43ab2bb99c98759749972181daa0eb56a9
Parents: 698989d
Author: Gour Saha 
Authored: Fri Oct 14 17:40:51 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../api/impl/ApplicationApiService.java | 10 --
 .../org/apache/slider/client/SliderClient.java  | 33 ++--
 .../AbstractClusterBuildingActionArgs.java  |  5 +++
 .../slider/common/params/ActionThawArgs.java|  6 
 .../apache/slider/common/params/Arguments.java  |  1 +
 5 files changed, 36 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf6a2e43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 21cf113..73df4a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -347,7 +347,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 if (queueName != null && queueName.trim().length() > 0) {
   createArgs.queue = queueName.trim();
 }
-
+createArgs.lifetime = application.getLifetime();
 return invokeSliderClientRunnable(new 
SliderClientContextRunnable() {
   @Override
   public String run(SliderClient sliderClient) throws YarnException,
@@ -1246,13 +1246,17 @@ public class ApplicationApiService implements 
ApplicationApi {
 });
   }
 
-  private Response startSliderApplication(final String appName)
+  private Response startSliderApplication(final String appName, Application 
app)
   throws IOException, YarnException, InterruptedException {
 return invokeSliderClientRunnable(new 
SliderClientContextRunnable() {
   @Override
   public Response run(SliderClient sliderClient) throws YarnException,
   IOException, InterruptedException {
 ActionThawArgs thawArgs = new ActionThawArgs();
+if (app.getLifetime() == null) {
+  app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+}
+thawArgs.lifetime = app.getLifetime();
 int returnCode = sliderClient.actionThaw(appName, thawArgs);
 if (returnCode == 0) {
   logger.info("Successfully started application {}", appName);
@@ -1344,7 +1348,7 @@ public class ApplicationApiService implements 
ApplicationApi {
   try {
 int livenessCheck = getSliderList(appName);
 if (livenessCheck != 0) {
-  return startSliderApplication(appName);
+  return startSliderApplication(appName, updateAppData);
 } else {
   logger.info("Application {} is already running", appName);
   ApplicationStatus applicationStatus = new ApplicationStatus();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf6a2e43/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fe4f1d2..2840c4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.util.Shell;
 

[72/74] [abbrv] hadoop git commit: YARN-5828. Native services client errors out when config formats are uppercase. Contributed by Billie Rinaldi

[jianhe]

YARN-5828. Native services client errors out when config formats are uppercase. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e9deefd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e9deefd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e9deefd

Branch: refs/heads/yarn-native-services
Commit: 0e9deefd258cd94c3d250d0b35a09e31c629e7a2
Parents: 8a9d70c
Author: Gour Saha 
Authored: Thu Nov 3 18:15:44 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../org/apache/slider/core/registry/docstore/ConfigFormat.java   | 4 +++-
 .../src/main/java/org/apache/slider/providers/ProviderUtils.java | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e9deefd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index ddab606..723b975 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -18,6 +18,8 @@
 
 package org.apache.slider.core.registry.docstore;
 
+import java.util.Locale;
+
 public enum ConfigFormat {
 
   JSON("json"),
@@ -51,7 +53,7 @@ public enum ConfigFormat {
*/
   public static ConfigFormat resolve(String type) {
 for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type)) {
+  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
 return format;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e9deefd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index c5e6782..39986c1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -611,7 +611,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
   }
   ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
   if (configFormat == null) {
-throw new BadConfigException("Config format " + configFormat +
+throw new BadConfigException("Config format " + configFileType +
 " doesn't exist");
   }
   localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[41/74] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be43905b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
deleted file mode 100644
index 373d64d..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ /dev/null
@@ -1,34473 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: SliderClusterMessages.proto
-
-package org.apache.slider.api.proto;
-
-public final class Messages {
-  private Messages() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  public interface RoleInstanceStateOrBuilder
-  extends com.google.protobuf.MessageOrBuilder {
-
-// required string name = 1;
-/**
- * required string name = 1;
- */
-boolean hasName();
-/**
- * required string name = 1;
- */
-java.lang.String getName();
-/**
- * required string name = 1;
- */
-com.google.protobuf.ByteString
-getNameBytes();
-
-// optional string role = 2;
-/**
- * optional string role = 2;
- */
-boolean hasRole();
-/**
- * optional string role = 2;
- */
-java.lang.String getRole();
-/**
- * optional string role = 2;
- */
-com.google.protobuf.ByteString
-getRoleBytes();
-
-// required uint32 state = 4;
-/**
- * required uint32 state = 4;
- */
-boolean hasState();
-/**
- * required uint32 state = 4;
- */
-int getState();
-
-// required uint32 exitCode = 5;
-/**
- * required uint32 exitCode = 5;
- */
-boolean hasExitCode();
-/**
- * required uint32 exitCode = 5;
- */
-int getExitCode();
-
-// optional string command = 6;
-/**
- * optional string command = 6;
- */
-boolean hasCommand();
-/**
- * optional string command = 6;
- */
-java.lang.String getCommand();
-/**
- * optional string command = 6;
- */
-com.google.protobuf.ByteString
-getCommandBytes();
-
-// optional string diagnostics = 7;
-/**
- * optional string diagnostics = 7;
- */
-boolean hasDiagnostics();
-/**
- * optional string diagnostics = 7;
- */
-java.lang.String getDiagnostics();
-/**
- * optional string diagnostics = 7;
- */
-com.google.protobuf.ByteString
-getDiagnosticsBytes();
-
-// repeated string output = 8;
-/**
- * repeated string output = 8;
- */
-java.util.List
-getOutputList();
-/**
- * repeated string output = 8;
- */
-int getOutputCount();
-/**
- * repeated string output = 8;
- */
-java.lang.String getOutput(int index);
-/**
- * repeated string output = 8;
- */
-com.google.protobuf.ByteString
-getOutputBytes(int index);
-
-// repeated string environment = 9;
-/**
- * repeated string environment = 9;
- */
-java.util.List
-getEnvironmentList();
-/**
- * repeated string environment = 9;
- */
-int getEnvironmentCount();
-/**
- * repeated string environment = 9;
- */
-java.lang.String getEnvironment(int index);
-/**
- * repeated string environment = 9;
- */
-com.google.protobuf.ByteString
-getEnvironmentBytes(int index);
-
-// required uint32 roleId = 10;
-/**
- * required uint32 roleId = 10;
- */
-boolean hasRoleId();
-/**
- * required uint32 roleId = 10;
- */
-int getRoleId();
-
-// required bool released = 11;
-/**
- * required bool released = 11;
- */
-boolean hasReleased();
-/**
- * required bool released = 11;
- */
-boolean getReleased();
-
-// required int64 createTime = 12;
-/**
- * required int64 createTime = 12;
- */
-boolean hasCreateTime();
-/**
- * required int64 createTime = 12;
- */
-long getCreateTime();
-
-// required int64 startTime = 13;
-/**
- * required int64 startTime = 13;
- */
-boolean hasStartTime();
-/**
- * required int64 startTime = 13;
- */
-long getStartTime();
-
-// required string host = 14;
-/**
- * required string host = 14;
- */
-boolean hasHost();
-/**
- * required string host = 14;
- */
-java.lang.String getHost();
-/**
- * required string host = 14;
- */
-

[49/74] [abbrv] hadoop git commit: YARN-5958. Fix ASF license warnings for slider core module. Contributed by Billie Rinaldi

[jianhe]

YARN-5958. Fix ASF license warnings for slider core module. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8defcbbc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8defcbbc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8defcbbc

Branch: refs/heads/yarn-native-services
Commit: 8defcbbc116915d51c95345b41cc0c6578ce6ded
Parents: 1abf917
Author: Gour Saha 
Authored: Thu Dec 1 17:45:44 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../hadoop-yarn-slider-core/pom.xml | 61 +---
 .../src/license/THIRD-PARTY.properties  | 33 ---
 2 files changed, 14 insertions(+), 80 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8defcbbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 66e9ee9..10cf6b1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -78,6 +78,20 @@
 
   
 
+  
+org.apache.rat
+apache-rat-plugin
+
+  
+**/*.json
+src/main/resources/webapps/slideram/.keep
+
+
src/main/java/org/apache/slider/api/proto/Messages.java
+
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+  
+
+  
+
 
   
   
@@ -384,53 +398,6 @@
   
 
 
-
-
-  rat
-  
-
-
-  
-org.apache.rat
-apache-rat-plugin
-
-  
-check-licenses
-
-  check
-
-  
-
-
-  
-**/*.json
-src/test/python/agent.ini
-src/test/python/version
-**/THIRD-PARTY.properties
-src/main/resources/webapps/slideram/.keep
-src/main/resources/webapps/slideragent/.keep
-
src/main/resources/webapps/static/yarn.dt.plugins.js
-
-
src/main/resources/webapps/static/dt-1.9.4/**
-
-
src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js
-
-
src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js
-
src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css
-
-
src/main/resources/webapps/static/jt/jquery.jstree.js
-
-
src/main/java/org/apache/slider/api/proto/Messages.java
-
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
-
src/test/app_packages/test_am_config/resources/test.template
-
src/test/app_packages/test_am_config/test_archive/testfile
-  
-
-  
-
-  
-
-
   
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8defcbbc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
deleted file mode 100644
index 1abd56e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
-#---
-# Already used licenses in project :
-# - Apache License
-# - BSD
-# - CDDL + GPLv2 with classpath exception
-# - CDDL 1.1
-# - CDDL License
-# - CDDL+GPL
-# - Common Public License Version 1.0
-# - Eclipse Public License - Version 1.0
-# - GNU Lesser General Public License (LGPL), Version 2.1
-# - GNU Lesser General Public License, Version 2.1
-# - GPL2 w/ CPE
-# - MIT License
-# - MPL 1.1
-# - New BSD License
-# - Public 

[45/74] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
new file mode 100644
index 000..a3780cc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or application. The resource specified at
+ * the app (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu  memory are exepected. It raises a validation
+ * exception otherwise.
+ **/
+
+@ApiModel(description = "Resource determines the amount of resources (vcores, 
memory, network, etc.) usable by a container. This field determines the 
resource to be applied for all the containers of a component or application. 
The resource specified at the app (or global) level can be overriden at the 
component level. Only one of profile OR cpu & memory are exepected. It raises a 
validation exception otherwise.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource {
+  private static final long serialVersionUID = -6431667797380250037L;
+
+  private String profile = null;
+  private Integer cpus = null;
+  private String memory = null;
+
+  /**
+   * Each resource profile has a unique id which is associated with a
+   * cluster-level predefined memory, cpus, etc.
+   **/
+  public Resource profile(String profile) {
+this.profile = profile;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Each resource profile has a 
unique id which is associated with a cluster-level predefined memory, cpus, 
etc.")
+  @JsonProperty("profile")
+  public String getProfile() {
+return profile;
+  }
+
+  public void setProfile(String profile) {
+this.profile = profile;
+  }
+
+  /**
+   * Amount of vcores allocated to each container (optional but overrides cpus
+   * in profile if specified).
+   **/
+  public Resource cpus(Integer cpus) {
+this.cpus = cpus;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Amount of vcores allocated to 
each container (optional but overrides cpus in profile if specified).")
+  @JsonProperty("cpus")
+  public Integer getCpus() {
+return cpus;
+  }
+
+  public void setCpus(Integer cpus) {
+this.cpus = cpus;
+  }
+
+  /**
+   * Amount of memory allocated to each container (optional but overrides 
memory
+   * in profile if specified). Currently accepts only an integer value and
+   * default unit is in MB.
+   **/
+  public Resource memory(String memory) {
+this.memory = memory;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Amount of memory allocated to 
each container (optional but overrides memory in profile if specified). 
Currently accepts only an integer value and default unit is in MB.")
+  @JsonProperty("memory")
+  public String getMemory() {
+return memory;
+  }
+
+  public void setMemory(String memory) {
+this.memory = memory;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != 

[69/74] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

YARN-5909. Remove agent related code in slider AM. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/038561ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/038561ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/038561ca

Branch: refs/heads/yarn-native-services
Commit: 038561caec133426fe8e078c200d09f04d2c83d7
Parents: 4e8d42b
Author: Jian He 
Authored: Tue Nov 29 13:06:45 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 hadoop-project/pom.xml  |5 -
 .../services/webapp/ApplicationApiWebApp.java   |2 +-
 .../hadoop-yarn-slider-core/pom.xml |   23 +-
 .../apache/slider/common/tools/SliderUtils.java |1 -
 .../providers/AbstractProviderService.java  |   21 +-
 .../slider/providers/ProviderService.java   |9 +-
 .../providers/agent/AgentClientProvider.java|  713 -
 .../providers/agent/AgentLaunchParameter.java   |  130 -
 .../providers/agent/AgentProviderFactory.java   |   47 -
 .../providers/agent/AgentProviderService.java   | 2850 --
 .../slider/providers/agent/AgentRoles.java  |   38 -
 .../slider/providers/agent/AgentUtils.java  |  150 -
 .../apache/slider/providers/agent/Command.java  |   59 -
 .../slider/providers/agent/CommandResult.java   |   40 -
 .../providers/agent/ComponentCommandOrder.java  |  225 --
 .../providers/agent/ComponentInstanceState.java |  340 ---
 .../providers/agent/ComponentTagProvider.java   |  127 -
 .../slider/providers/agent/ContainerState.java  |   41 -
 .../providers/agent/HeartbeatMonitor.java   |  130 -
 .../apache/slider/providers/agent/State.java|  199 --
 .../application/metadata/AbstractComponent.java |   80 -
 .../metadata/AbstractMetainfoParser.java|  130 -
 .../metadata/AbstractMetainfoSchema.java|   69 -
 .../metadata/AddonPackageMetainfoParser.java|   53 -
 .../agent/application/metadata/Application.java |  193 --
 .../metadata/ApplicationPackage.java|   69 -
 .../application/metadata/CommandOrder.java  |   61 -
 .../application/metadata/CommandScript.java |   72 -
 .../agent/application/metadata/Component.java   |  217 --
 .../application/metadata/ComponentCommand.java  |   85 -
 .../application/metadata/ComponentExport.java   |   54 -
 .../metadata/ComponentsInAddonPackage.java  |   26 -
 .../agent/application/metadata/ConfigFile.java  |   59 -
 .../application/metadata/DefaultConfig.java |   39 -
 .../metadata/DefaultConfigParser.java   |   54 -
 .../application/metadata/DockerContainer.java   |  187 --
 .../metadata/DockerContainerInputFile.java  |   50 -
 .../metadata/DockerContainerMount.java  |   60 -
 .../metadata/DockerContainerPort.java   |   66 -
 .../agent/application/metadata/Export.java  |   61 -
 .../agent/application/metadata/ExportGroup.java |   71 -
 .../agent/application/metadata/Metainfo.java|  118 -
 .../application/metadata/MetainfoParser.java|   97 -
 .../agent/application/metadata/OSPackage.java   |   51 -
 .../agent/application/metadata/OSSpecific.java  |   57 -
 .../agent/application/metadata/Package.java |   60 -
 .../application/metadata/PropertyInfo.java  |   54 -
 .../agent/application/metadata/Validate.java|   27 -
 .../org/apache/slider/providers/agent/todo.md   |   22 -
 .../server/appmaster/SliderAppMaster.java   |  140 +-
 .../server/appmaster/web/AgentService.java  |   37 -
 .../slider/server/appmaster/web/WebAppApi.java  |7 -
 .../server/appmaster/web/WebAppApiImpl.java |6 -
 .../appmaster/web/rest/AMWebServices.java   |1 -
 .../web/rest/agent/AgentCommandType.java|   23 -
 .../appmaster/web/rest/agent/AgentEnv.java  |  376 ---
 .../appmaster/web/rest/agent/AgentResource.java |  118 -
 .../web/rest/agent/AgentRestOperations.java |   28 -
 .../appmaster/web/rest/agent/AgentWebApp.java   |  258 --
 .../web/rest/agent/AgentWebServices.java|   40 -
 .../appmaster/web/rest/agent/CommandReport.java |  207 --
 .../web/rest/agent/ComponentStatus.java |  129 -
 .../appmaster/web/rest/agent/DiskInfo.java  |  128 -
 .../web/rest/agent/ExecutionCommand.java|  310 --
 .../appmaster/web/rest/agent/HeartBeat.java |  149 -
 .../web/rest/agent/HeartBeatResponse.java   |  147 -
 .../appmaster/web/rest/agent/HostInfo.java  |  398 ---
 .../appmaster/web/rest/agent/HostStatus.java|   63 -
 .../appmaster/web/rest/agent/Register.java  |  193 --
 .../web/rest/agent/RegistrationCommand.java |   43 -
 .../web/rest/agent/RegistrationResponse.java|  133 -
 .../web/rest/agent/RegistrationStatus.java  |   22 -
 .../appmaster/web/rest/agent/StatusCommand.java |  152 -
 .../agent/TestAgentClientProvider.java  

[66/74] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
deleted file mode 100644
index 78bb8c1..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Component defined in master package metainfo.json
- */
-public class Component extends AbstractComponent {
-
-  String category = CATEGORY_MASTER;
-  String publishConfig = Boolean.FALSE.toString();
-  String minInstanceCount = "0";
-  String maxInstanceCount;
-  String autoStartOnFailure = Boolean.FALSE.toString();
-  String appExports;
-  String compExports;
-  String type = TYPE_STANDARD;
-  List componentExports = new ArrayList<>();
-  List dockerContainers = new ArrayList<>();
-  List configFiles = new ArrayList<>();
-
-  public Component() {
-  }
-
-  public String getType() {
-return type;
-  }
-
-  public void setType(String type) {
-this.type = type;
-  }
-
-  public String getCategory() {
-return category;
-  }
-
-  public void setCategory(String category) {
-this.category = category;
-  }
-
-  public String getPublishConfig() {
-return publishConfig;
-  }
-
-  public void setPublishConfig(String publishConfig) {
-this.publishConfig = publishConfig;
-  }
-
-  public String getAutoStartOnFailure() {
-return autoStartOnFailure;
-  }
-
-  public void setAutoStartOnFailure(String autoStartOnFailure) {
-this.autoStartOnFailure = autoStartOnFailure;
-  }
-
-  public String getAppExports() {
-return appExports;
-  }
-
-  public void setAppExports(String appExports) {
-this.appExports = appExports;
-  }
-
-  public String getCompExports() {
-return compExports;
-  }
-
-  public void setCompExports(String compExports) {
-this.compExports = compExports;
-  }
-
-  public String getMinInstanceCount() {
-return minInstanceCount;
-  }
-  
-  @JsonProperty("dockerContainers")
-  public List getDockerContainers() {
- return this.dockerContainers;
-  }
-  
-  public Boolean getAutoStartOnFailureBoolean() {
-if (SliderUtils.isUnset(getAutoStartOnFailure())) {
-  return Boolean.FALSE;
-}
-
-return Boolean.parseBoolean(getAutoStartOnFailure());
-  }
-
-  public int getMinInstanceCountInt() throws BadConfigException {
-if (SliderUtils.isUnset(minInstanceCount)) {
-  return 0;
-}
-
-try {
-  return Integer.parseInt(minInstanceCount);
-} catch (NumberFormatException nfe) {
-  throw new BadConfigException(nfe, "Invalid value for minInstanceCount 
for %s", name);
-}
-  }
-
-  public int getMaxInstanceCountInt() throws BadConfigException {
-if (SliderUtils.isUnset(maxInstanceCount)) {
-  return Integer.MAX_VALUE;
-}
-
-try {
-  return Integer.parseInt(maxInstanceCount);
-} catch (NumberFormatException nfe) {
-  throw new BadConfigException(nfe, "Invalid value for maxInstanceCount 
for %s", name);
-}
-  }
-
-  public void setMinInstanceCount(String minInstanceCount) {
-this.minInstanceCount = minInstanceCount;
-  }
-
-  public String getMaxInstanceCount() {
-return 

[68/74] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
deleted file mode 100644
index 499812e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ /dev/null
@@ -1,2850 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.MonitorDetail;
-import org.apache.slider.providers.ProviderCore;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import 
org.apache.slider.providers.agent.application.metadata.AbstractComponent;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.apache.slider.providers.agent.application.metadata.CommandScript;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
-import org.apache.slider.providers.agent.application.metadata.ComponentExport;
-import 
org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import 

[67/74] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
deleted file mode 100644
index 281895a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AgentRoles {
-
-  /**
-   * List of roles Agent provider does not have any roles by default. All 
roles are read from the application
-   * specification.
-   */
-  protected static final List ROLES =
-  new ArrayList();
-
-  public static List getRoles() {
-return ROLES;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
deleted file mode 100644
index 23e05a3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import 
org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
-import 
org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import 
org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-

[70/74] [abbrv] hadoop git commit: YARN-5941. Slider handles "per.component" for multiple components incorrectly. Contributed by Billie Rinaldi

[jianhe]

YARN-5941. Slider handles "per.component" for multiple components incorrectly. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40529b07
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40529b07
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40529b07

Branch: refs/heads/yarn-native-services
Commit: 40529b0722de3c264d12b3a174f23d95f673f705
Parents: 0d69d71
Author: Gour Saha 
Authored: Wed Nov 30 14:00:22 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../java/org/apache/slider/api/OptionKeys.java  | 14 ++
 .../org/apache/slider/common/SliderKeys.java|  2 --
 .../apache/slider/providers/ProviderUtils.java  | 20 
 3 files changed, 30 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/40529b07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 434b1d9..988627d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -53,8 +53,22 @@ public interface OptionKeys extends InternalKeys {
* Prefix for export options: {@value}
*/
   String EXPORT_PREFIX = "export.";
+  /**
+   * Type suffix for config file and package options: {@value}
+   */
   String TYPE_SUFFIX = ".type";
+  /**
+   * Name suffix for config file and package options: {@value}
+   */
   String NAME_SUFFIX = ".name";
+  /**
+   * Per component suffix for config file options: {@value}
+   */
+  String PER_COMPONENT = ".per.component";
+  /**
+   * Per group suffix for config file options: {@value}
+   */
+  String PER_GROUP = ".per.group";
 
   /**
* Zookeeper quorum host list: {@value}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40529b07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 3d25d33..adf40ce 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -304,8 +304,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
 
   String APP_RESOURCES = "application.resources";
   String APP_RESOURCES_DIR = "app/resources";
-  String PER_COMPONENT = "per.component";
-  String PER_GROUP = "per.group";
 
   String APP_PACKAGES_DIR = "app/packages";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40529b07/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 39986c1..bc237f5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -614,8 +614,16 @@ public class ProviderUtils implements RoleKeys, 

[62/74] [abbrv] hadoop git commit: YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by Billie Rinaldi

[jianhe]

YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/813e1325
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/813e1325
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/813e1325

Branch: refs/heads/yarn-native-services
Commit: 813e132515855e3e22641089974328838cfb2983
Parents: cf6a2e4
Author: Jian He 
Authored: Sun Oct 16 17:01:09 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../org/apache/slider/client/SliderClient.java  | 72 
 .../slider/core/buildutils/InstanceBuilder.java |  4 +
 .../apache/slider/providers/ProviderUtils.java  | 62 --
 .../providers/docker/DockerClientProvider.java  |  4 +-
 .../providers/docker/DockerProviderService.java | 87 
 5 files changed, 164 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/813e1325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 2840c4b..94e51e5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -178,6 +178,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
+import java.io.Console;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -918,57 +919,56 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   return;
 }
 
-BufferedReader br = null;
-try {
-  for (Entry cred : tree.credentials.entrySet()) {
-String provider = cred.getKey()
-.replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
-.replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
-List aliases = cred.getValue();
-if (aliases == null || aliases.isEmpty()) {
-  continue;
-}
-Configuration c = new Configuration(conf);
-c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
-CredentialProvider credentialProvider = 
CredentialProviderFactory.getProviders(c).get(0);
-Set existingAliases = new 
HashSet<>(credentialProvider.getAliases());
-for (String alias : aliases) {
-  if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
-log.info("Credentials for " + alias + " found in " + provider);
-  } else {
-if (br == null) {
-  br = new BufferedReader(new InputStreamReader(System.in));
-}
-char[] pass = readPassword(alias, br);
-credentialProvider.createCredentialEntry(alias, pass);
-credentialProvider.flush();
-Arrays.fill(pass, ' ');
+Console console = System.console();
+for (Entry cred : tree.credentials.entrySet()) {
+  String provider = cred.getKey()
+  .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
+  .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
+  List aliases = cred.getValue();
+  if (aliases == null || aliases.isEmpty()) {
+continue;
+  }
+  Configuration c = new Configuration(conf);
+  c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+  CredentialProvider credentialProvider = 
CredentialProviderFactory.getProviders(c).get(0);
+  Set existingAliases = new 
HashSet<>(credentialProvider.getAliases());
+  for (String alias : aliases) {
+if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
+  log.info("Credentials for " + alias + " found in " + provider);
+} else {
+  if (console == null) {
+throw new IOException("Unable to input password for " + alias +
+" because System.console() is null; provider " + provider +
+" must be populated manually");
   }
+  char[] pass = readPassword(alias, console);
+  

[61/74] [abbrv] hadoop git commit: YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha

[jianhe]

YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38063447
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38063447
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38063447

Branch: refs/heads/yarn-native-services
Commit: 3806344721b82609baa996176996573369ca40aa
Parents: 689cbc4
Author: Jian He 
Authored: Wed Oct 12 13:27:53 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 416 +++
 1 file changed, 416 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38063447/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
new file mode 100644
index 000..6169fcd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -0,0 +1,416 @@
+# Hadoop YARN REST APIs for services v1 spec in YAML
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+swagger: '2.0'
+info:
+  title: "[YARN-4793] Simplified API layer for services and beyond"
+  description: |
+Bringing a new service on YARN today is not a simple experience. The APIs 
of existing frameworks are either too low level (native YARN), require writing 
new code (for frameworks with programmatic APIs) or writing a complex spec (for 
declarative frameworks). In addition to building critical building blocks 
inside YARN (as part of other efforts at 
link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a 
need for simplifying the user facing story for building services. Experience of 
projects like Apache Slider running real-life services like HBase, Storm, 
Accumulo, Solr etc, gives us some very good insights on how simplified APIs for 
services should look like.
+
+
+To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component apps to 
complex multi-component assemblies needing orchestration.
+
+
+We should also look at making this a unified REST based entry point for 
other important features like resource-profile management 
(link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), 
package-definitions' lifecycle-management and service-discovery 
(link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]).
 We also need to flesh out its relation to our present much lower level REST 
APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN 
for application-submission and management.
+
+
+This document spotlights on this specification. In most of the cases, the 
application owner will not be forced to make any changes to their application. 
This is primarily true if the application is packaged with containerization 
technologies like docker. Irrespective of how complex the application is, there 
will be hooks provided at appropriate layers to allow pluggable and 
customizable application behavior.
+
+  version: "1.0.0"
+  license:
+name: Apache 2.0
+url: http://www.apache.org/licenses/LICENSE-2.0.html
+# the domain of the service
+host: host.mycompany.com
+# array of all schemes that your API supports
+schemes:
+  - http
+  - https
+# will be prefixed to all paths

[64/74] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/038561ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
deleted file mode 100644
index 0150079..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.providers.agent.State;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/** Data model for agent to send heartbeat to ambari and/or app master. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class Register {
-  private int responseId = -1;
-  private long timestamp;
-  private String label;
-  private int currentPingPort;
-  private HostInfo hardwareProfile;
-  private String publicHostname;
-  private String tags;
-  private AgentEnv agentEnv;
-  private String agentVersion;
-  private State actualState;
-  private State expectedState;
-  private Map allocatedPorts;
-  private Map logFolders;
-  private String pkg;
-  private String appVersion;
-
-  @JsonProperty("responseId")
-  public int getResponseId() {
-return responseId;
-  }
-
-  @JsonProperty("responseId")
-  public void setResponseId(int responseId) {
-this.responseId = responseId;
-  }
-
-  public long getTimestamp() {
-return timestamp;
-  }
-
-  public void setTimestamp(long timestamp) {
-this.timestamp = timestamp;
-  }
-
-  public String getLabel() {
-return label;
-  }
-
-  public void setLabel(String label) {
-this.label = label;
-  }
-
-  public String getTags() {
-return tags;
-  }
-
-  public void setTags(String tags) {
-this.tags = tags;
-  }
-
-  public HostInfo getHardwareProfile() {
-return hardwareProfile;
-  }
-
-  public void setHardwareProfile(HostInfo hardwareProfile) {
-this.hardwareProfile = hardwareProfile;
-  }
-
-  public String getPublicHostname() {
-return publicHostname;
-  }
-
-  public void setPublicHostname(String name) {
-this.publicHostname = name;
-  }
-
-  public AgentEnv getAgentEnv() {
-return agentEnv;
-  }
-
-  public void setAgentEnv(AgentEnv env) {
-this.agentEnv = env;
-  }
-
-  public String getAgentVersion() {
-return agentVersion;
-  }
-
-  public void setAgentVersion(String agentVersion) {
-this.agentVersion = agentVersion;
-  }
-
-  public int getCurrentPingPort() {
-return currentPingPort;
-  }
-
-  public void setCurrentPingPort(int currentPingPort) {
-this.currentPingPort = currentPingPort;
-  }
-
-  public State getActualState() {
-return actualState;
-  }
-
-  public void setActualState(State actualState) {
-this.actualState = actualState;
-  }
-
-  public State getExpectedState() {
-return expectedState;
-  }
-
-  public void setExpectedState(State expectedState) {
-this.expectedState = expectedState;
-  }
-
-  /** @return the allocated ports, or null if none are present */
-  @JsonProperty("allocatedPorts")
-  public Map getAllocatedPorts() {
-return allocatedPorts;
-  }
-
-  /** @param ports allocated ports */
-  @JsonProperty("allocatedPorts")
-  public void setAllocatedPorts(Map ports) {
-this.allocatedPorts = ports;
-  }
-
-  /** @return the log folders, or null if none are present */
-  

[71/74] [abbrv] hadoop git commit: YARN-5680. Add 2 new fields in Slider status output - image-name and is-privileged-container. Contributed by Billie Rinaldi

[jianhe]

YARN-5680. Add 2 new fields in Slider status output - image-name and 
is-privileged-container. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b10d4cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b10d4cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b10d4cb

Branch: refs/heads/yarn-native-services
Commit: 7b10d4cba4c4ae23b594c5584db26c55604f8676
Parents: 23963b5
Author: Gour Saha 
Authored: Tue Oct 25 20:00:27 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../org/apache/slider/providers/docker/DockerKeys.java  |  1 +
 .../slider/providers/docker/DockerProviderService.java  |  2 +-
 .../apache/slider/server/appmaster/state/AppState.java  | 12 
 3 files changed, 14 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b10d4cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
index 40b73a2..0e1d288 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
@@ -26,6 +26,7 @@ public interface DockerKeys {
   String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
 
   String DEFAULT_DOCKER_NETWORK = "bridge";
+  Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
 
   String OUT_FILE = "stdout.txt";
   String ERR_FILE = "stderr.txt";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b10d4cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index af36620..cc319ee 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -131,7 +131,7 @@ public class DockerProviderService extends 
AbstractProviderService implements
 launcher.setDockerNetwork(appConf.getComponentOpt(roleGroup, 
DOCKER_NETWORK,
 DEFAULT_DOCKER_NETWORK));
 launcher.setRunPrivilegedContainer(appConf.getComponentOptBool(roleGroup,
-DOCKER_USE_PRIVILEGED, false));
+DOCKER_USE_PRIVILEGED, DEFAULT_DOCKER_USE_PRIVILEGED));
 
 // Set the environment
 Map standardTokens = providerUtils.getStandardTokenMap(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b10d4cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 49e7b78..6db375d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -91,6 +91,9 @@ 

[54/74] [abbrv] hadoop git commit: YARN-5943. Write native services container stderr file to log directory. Contributed by Billie Rinaldi

[jianhe]

YARN-5943. Write native services container stderr file to log directory. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d69d714
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d69d714
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d69d714

Branch: refs/heads/yarn-native-services
Commit: 0d69d714f064431ff32550f83690e6a160471d8b
Parents: fdea5e1
Author: Gour Saha 
Authored: Wed Nov 30 10:30:39 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../org/apache/slider/providers/docker/DockerProviderService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d69d714/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index cc319ee..e4a7cdf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -193,7 +193,8 @@ public class DockerProviderService extends 
AbstractProviderService implements
 "/bin/bash"));
 
 operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
-+ OUT_FILE + " 2>" + ERR_FILE);
++ OUT_FILE + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
++ ERR_FILE);
 
 launcher.addCommand(operation.build());
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[51/74] [abbrv] hadoop git commit: YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha

[jianhe]

YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/698989d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/698989d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/698989d4

Branch: refs/heads/yarn-native-services
Commit: 698989d4d4556b8489dba757c1ec7ea615fffde1
Parents: 713018d
Author: Jian He 
Authored: Fri Oct 14 13:47:38 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../services/api/impl/ApplicationApiService.java| 16 ++--
 .../hadoop/yarn/services/resource/Application.java  |  6 +++---
 .../hadoop/yarn/services/resource/Artifact.java |  4 +++-
 .../hadoop/yarn/services/resource/Component.java|  4 +++-
 .../hadoop/yarn/services/resource/ConfigFile.java   |  4 +++-
 .../yarn/services/resource/Configuration.java   |  4 +++-
 .../hadoop/yarn/services/resource/Container.java|  6 +++---
 .../yarn/services/resource/PlacementPolicy.java |  4 +++-
 .../yarn/services/resource/ReadinessCheck.java  |  4 +++-
 .../hadoop/yarn/services/resource/Resource.java |  2 +-
 10 files changed, 39 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/698989d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 0a62629..21cf113 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -276,7 +276,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 
 // If it is a simple app with no components, then create a default 
component
 if (application.getComponents() == null) {
-  application.setComponents(getDefaultComponentAsList());
+  application.setComponents(getDefaultComponentAsList(application));
 }
 
 // Application lifetime if not specified, is set to unlimited lifetime
@@ -1029,7 +1029,8 @@ public class ApplicationApiService implements 
ApplicationApi {
 // end-users point of view, is out of scope of the REST API. Also, this
 // readiness has nothing to do with readiness-check defined at the 
component
 // level (which is used for dependency resolution of component DAG).
-if (totalNumberOfIpAssignedContainers == 
totalExpectedNumberOfRunningContainers) {
+if (totalNumberOfIpAssignedContainers
+.longValue() == totalExpectedNumberOfRunningContainers.longValue()) {
   app.setState(ApplicationState.READY);
 }
 logger.info("Application = {}", app);
@@ -1389,6 +1390,17 @@ public class ApplicationApiService implements 
ApplicationApi {
 return Response.status(Status.NO_CONTENT).build();
   }
 
+  // create default component and initialize with app level global values
+  private List getDefaultComponentAsList(Application app) {
+List comps = getDefaultComponentAsList();
+Component comp = comps.get(0);
+comp.setArtifact(app.getArtifact());
+comp.setResource(app.getResource());
+comp.setNumberOfContainers(app.getNumberOfContainers());
+comp.setLaunchCommand(app.getLaunchCommand());
+return comps;
+  }
+
   private List getDefaultComponentAsList() {
 Component comp = new Component();
 comp.setName(DEFAULT_COMPONENT_NAME);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/698989d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index 719bf95..ed65ad2 100644
--- 

[52/74] [abbrv] hadoop git commit: YARN-5808. Add gc log options to the yarn daemon script when starting services-api. Contributed by Billie Rinaldi

[jianhe]

YARN-5808. Add gc log options to the yarn daemon script when starting 
services-api. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e8d42bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e8d42bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e8d42bf

Branch: refs/heads/yarn-native-services
Commit: 4e8d42bfaace7917d77ac391cf45b656fda7d762
Parents: 0e9deef
Author: Gour Saha 
Authored: Thu Nov 10 11:35:02 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn | 16 
 hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 12 
 2 files changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e8d42bf/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 26d54b8..2396a7a 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,7 @@ function hadoop_usage
   hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" "admin tools"
   hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
-  hadoop_add_subcommand "services-api" "run slider services api"
+  hadoop_add_subcommand "servicesapi" "run slider services api"
   hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager 
daemon"
   hadoop_add_subcommand "slider" "run a slider app"
   hadoop_add_subcommand "timelinereader" "run the timeline reader server"
@@ -144,20 +144,20 @@ function yarncmd_case
 scmadmin)
   HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
 ;;
-services-api)
+servicesapi)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 sharedcachemanager)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -166,15 +166,15 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
 slider)
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   HADOOP_CLASSNAME='org.apache.slider.Slider'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 timelinereader)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e8d42bf/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh 
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index d003adb..3828897 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -136,3 +136,15 @@
 # See ResourceManager for some examples
 #
 #export YARN_SHAREDCACHEMANAGER_OPTS=
+
+###
+# Services API specific parameters
+###
+# Specify the JVM options to be used when starting the services API.
+#
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails 
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps 

[55/74] [abbrv] hadoop git commit: YARN-5775. Convert enums in swagger definition to uppercase. Contributed by Gour Saha

[jianhe]

YARN-5775. Convert enums in swagger definition to uppercase. Contributed by 
Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23963b57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23963b57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23963b57

Branch: refs/heads/yarn-native-services
Commit: 23963b57d0f5c358510cfd4412f2043d6be31be3
Parents: 53e885d
Author: Billie Rinaldi 
Authored: Tue Oct 25 11:25:51 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 38 ++--
 1 file changed, 19 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/23963b57/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 6169fcd..7eb3196 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -241,10 +241,10 @@ definitions:
 type: string
 description: Artifact type, like docker, tarball, etc. (optional).
 enum:
-  - docker
-  - tarball
-  - application
-default: docker
+  - DOCKER
+  - TARBALL
+  - APPLICATION
+default: DOCKER
   uri:
 type: string
 description: Artifact location to support multiple artifact stores 
(optional).
@@ -303,7 +303,7 @@ definitions:
 type: string
 description: E.g. HTTP (YARN will perform a simple REST call at a 
regular interval and expect a 204 No content).
 enum:
-  - http
+  - HTTP
   uri:
 type: string
 description: Fully qualified REST uri endpoint.
@@ -335,13 +335,13 @@ definitions:
 type: string
 description: Config file in the standard format like xml, properties, 
json, yaml, template.
 enum:
-  - xml
-  - properties
-  - json
-  - yaml
-  - template
-  - env
-  - hadoop_xml
+  - XML
+  - PROPERTIES
+  - JSON
+  - YAML
+  - TEMPLATE
+  - ENV
+  - HADOOP_XML
   dest_file:
 type: string
 description: The absolute path that this configuration file should be 
mounted as, in the application container.
@@ -386,11 +386,11 @@ definitions:
 type: string
 description: enum of the state of the application
 enum:
-  - accepted
-  - started
-  - ready
-  - stopped
-  - failed
+  - ACCEPTED
+  - STARTED
+  - READY
+  - STOPPED
+  - FAILED
   ContainerState:
 description: The current state of the container of an application.
 properties:
@@ -398,8 +398,8 @@ definitions:
 type: string
 description: enum of the state of the container
 enum:
-  - init
-  - ready
+  - INIT
+  - READY
   ApplicationStatus:
 description: The current status of a submitted application, returned as a 
response to the GET API.
 properties:


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[38/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

YARN-5461. Initial code ported from slider-core module. (jianhe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31c4a419
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31c4a419
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31c4a419

Branch: refs/heads/yarn-native-services
Commit: 31c4a4198ec83637d17c7d986c1472ff10c49bb0
Parents: ef34bf2
Author: Jian He 
Authored: Wed Aug 3 00:18:01 2016 +0800
Committer: Jian He 
Committed: Tue Dec 13 14:44:52 2016 -0800

--
 .../hadoop-yarn-slider-core/pom.xml |   397 +
 .../src/assembly/executable-jar.xml |47 +
 .../src/license/THIRD-PARTY.properties  |33 +
 .../slider/server/avro/RoleHistoryRecord.avsc   |   114 +
 .../apache/hadoop/security/KerberosDiags.java   |   676 +
 .../src/main/java/org/apache/slider/Slider.java |52 +
 .../apache/slider/api/ClusterDescription.java   |   795 +
 .../slider/api/ClusterDescriptionKeys.java  |25 +
 .../api/ClusterDescriptionOperations.java   |93 +
 .../java/org/apache/slider/api/ClusterNode.java |   220 +
 .../org/apache/slider/api/InternalKeys.java |   199 +
 .../java/org/apache/slider/api/OptionKeys.java  |58 +
 .../org/apache/slider/api/ResourceKeys.java |   201 +
 .../java/org/apache/slider/api/RoleKeys.java|   116 +
 .../apache/slider/api/SliderApplicationApi.java |   167 +
 .../slider/api/SliderClusterProtocol.java   |   179 +
 .../java/org/apache/slider/api/StateValues.java |53 +
 .../java/org/apache/slider/api/StatusKeys.java  |   117 +
 .../org/apache/slider/api/proto/Messages.java   | 34473 +
 .../slider/api/proto/RestTypeMarshalling.java   |   319 +
 .../slider/api/proto/SliderClusterAPI.java  |  2293 ++
 .../types/ApplicationLivenessInformation.java   |53 +
 .../slider/api/types/ComponentInformation.java  |   107 +
 .../slider/api/types/ContainerInformation.java  |58 +
 .../slider/api/types/NodeEntryInformation.java  |78 +
 .../slider/api/types/NodeInformation.java   |59 +
 .../slider/api/types/NodeInformationList.java   |41 +
 .../slider/api/types/PingInformation.java   |47 +
 .../apache/slider/api/types/RoleStatistics.java |66 +
 .../api/types/SliderInstanceDescription.java|54 +
 .../slider/client/ClientRegistryBinder.java |   201 +
 .../org/apache/slider/client/ClientUtils.java   |   111 +
 .../org/apache/slider/client/SliderClient.java  |  4569 +++
 .../apache/slider/client/SliderClientAPI.java   |   368 +
 .../slider/client/SliderYarnClientImpl.java |   410 +
 .../apache/slider/client/TokensOperation.java   |   109 +
 .../client/ipc/SliderApplicationIpcClient.java  |   245 +
 .../client/ipc/SliderClusterOperations.java |   529 +
 .../slider/client/rest/BaseRestClient.java  |   152 +
 .../slider/client/rest/RestClientFactory.java   |89 +
 .../rest/SliderApplicationApiRestClient.java|   326 +
 .../org/apache/slider/common/Constants.java |35 +
 .../apache/slider/common/SliderExitCodes.java   |88 +
 .../org/apache/slider/common/SliderKeys.java|   278 +
 .../common/SliderXMLConfKeysForTesting.java |83 +
 .../apache/slider/common/SliderXmlConfKeys.java |   206 +
 .../common/params/AbstractActionArgs.java   |   178 +
 .../common/params/AbstractArgsDelegate.java |26 +
 .../AbstractClusterBuildingActionArgs.java  |   217 +
 .../common/params/ActionAMSuicideArgs.java  |44 +
 .../slider/common/params/ActionBuildArgs.java   |32 +
 .../slider/common/params/ActionClientArgs.java  |98 +
 .../slider/common/params/ActionCreateArgs.java  |59 +
 .../common/params/ActionDependencyArgs.java |65 +
 .../slider/common/params/ActionDestroyArgs.java |37 +
 .../common/params/ActionDiagnosticArgs.java |73 +
 .../slider/common/params/ActionEchoArgs.java|33 +
 .../slider/common/params/ActionExistsArgs.java  |47 +
 .../slider/common/params/ActionFlexArgs.java|54 +
 .../slider/common/params/ActionFreezeArgs.java  |56 +
 .../slider/common/params/ActionHelpArgs.java|51 +
 .../common/params/ActionInstallKeytabArgs.java  |57 +
 .../common/params/ActionInstallPackageArgs.java |58 +
 .../slider/common/params/ActionKDiagArgs.java   |86 +
 .../slider/common/params/ActionKeytabArgs.java  |81 +
 .../common/params/ActionKillContainerArgs.java  |37 +
 .../slider/common/params/ActionListArgs.java|74 +
 .../slider/common/params/ActionLookupArgs.java  |76 +
 .../slider/common/params/ActionNodesArgs.java   |71 +
 .../slider/common/params/ActionPackageArgs.java |81 +
 .../common/params/ActionRegistryArgs.java   |   218 +
 .../slider/common/params/ActionResolveArgs.java |   153 +
 .../common/params/ActionResourceArgs.java   |68 +
 

[43/74] [abbrv] hadoop git commit: YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi

[jianhe]

YARN-5690. Integrate native services modules into maven build. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f965dd19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f965dd19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f965dd19

Branch: refs/heads/yarn-native-services
Commit: f965dd19b35d0126a8d8da9fbb61327fec74ab7e
Parents: 05c15d7
Author: Gour Saha 
Authored: Thu Oct 27 08:50:36 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../resources/assemblies/hadoop-yarn-dist.xml   | 26 
 .../assemblies/hadoop-yarn-services-api.xml | 36 +++
 .../assemblies/hadoop-yarn-slider-dist.xml  | 30 +
 hadoop-project/pom.xml  |  6 ++
 hadoop-yarn-project/hadoop-yarn/bin/yarn| 30 +
 .../hadoop-yarn-services-api/pom.xml| 44 +++--
 .../conf/slideram-log4j.properties  | 68 
 .../hadoop-yarn-slider-core/pom.xml | 38 +++
 .../org/apache/slider/client/SliderClient.java  | 29 +++--
 .../org/apache/slider/common/SliderKeys.java|  2 +-
 .../apache/slider/common/tools/SliderUtils.java | 48 ++
 .../providers/agent/AgentClientProvider.java|  3 +-
 .../slideram/SliderAMClientProvider.java| 15 +++--
 .../TestPublishedConfigurationOutputter.java| 10 ++-
 14 files changed, 343 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f965dd19/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index c3f459c..552087c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -86,6 +86,32 @@
   
 
 
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target
+  
/share/hadoop/${hadoop.component}/sources
+  
+*-sources.jar
+  
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf
+  etc/hadoop
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}
+  
/share/hadoop/${hadoop.component}/lib/slider
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
+  
/share/hadoop/${hadoop.component}/sources
+  
+*-sources.jar
+  
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}
+  
/share/hadoop/${hadoop.component}/lib/services-api
+
+
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f965dd19/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
new file mode 100644
index 000..589f724
--- /dev/null
+++ 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
@@ -0,0 +1,36 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0;
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+  
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
 http://maven.apache.org/xsd/assembly-1.1.0.xsd;>
+  hadoop-yarn-services-api-dist
+  
+dir
+  
+  false
+  
+
+  false
+  
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base
+
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
+
com.fasterxml.jackson.module:jackson-module-jaxb-annotations
+io.swagger:swagger-annotations
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f965dd19/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
new file mode 100644
index 000..5de45a9
--- /dev/null
+++ 

[47/74] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

YARN-5610. Initial code for native services REST API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/689cbc40
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/689cbc40
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/689cbc40

Branch: refs/heads/yarn-native-services
Commit: 689cbc40c11cb032f8470c148507538a8d709881
Parents: b627324
Author: Jian He 
Authored: Tue Oct 11 11:36:57 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 hadoop-project/pom.xml  |   20 +
 .../dev-support/findbugs-exclude.xml|   20 +
 .../hadoop-yarn-services-api/pom.xml|  225 +++
 .../yarn/services/api/ApplicationApi.java   |   38 +
 .../api/impl/ApplicationApiService.java | 1527 ++
 .../yarn/services/resource/Application.java |  452 ++
 .../services/resource/ApplicationState.java |   25 +
 .../services/resource/ApplicationStatus.java|  147 ++
 .../hadoop/yarn/services/resource/Artifact.java |  155 ++
 .../yarn/services/resource/BaseResource.java|   48 +
 .../yarn/services/resource/Component.java   |  377 +
 .../yarn/services/resource/ConfigFile.java  |  190 +++
 .../yarn/services/resource/Configuration.java   |  147 ++
 .../yarn/services/resource/Container.java   |  256 +++
 .../yarn/services/resource/ContainerState.java  |   25 +
 .../hadoop/yarn/services/resource/Error.java|  125 ++
 .../yarn/services/resource/PlacementPolicy.java |   97 ++
 .../yarn/services/resource/ReadinessCheck.java  |  161 ++
 .../hadoop/yarn/services/resource/Resource.java |  149 ++
 .../yarn/services/utils/RestApiConstants.java   |   66 +
 .../services/utils/RestApiErrorMessages.java|   79 +
 .../services/webapp/ApplicationApiWebApp.java   |  127 ++
 .../src/main/resources/log4j-server.properties  |   76 +
 .../resources/webapps/services-rest-api/app |   16 +
 .../src/main/scripts/run_rest_service.sh|   28 +
 .../src/main/webapp/WEB-INF/web.xml |   36 +
 .../api/impl/TestApplicationApiService.java |  232 +++
 .../hadoop-yarn-applications/pom.xml|2 +-
 28 files changed, 4845 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 9ec24ea..8e8db00 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -129,6 +129,9 @@
 
 ${pom.version}
+
+1.5.4
+
1.4
   
 
   
@@ -1238,6 +1241,23 @@
   kerb-simplekdc
   1.0.0-RC2
 
+
+  
+io.swagger
+swagger-annotations
+${swagger-annotations-version}
+  
+  
+com.fasterxml.jackson.jaxrs
+jackson-jaxrs-json-provider
+${jackson2.version}
+  
+  
+org.apache.maven.doxia
+doxia-module-markdown
+${maven-doxia-module-markdown.version}
+  
+
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
new file mode 100644
index 000..b89146a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
new file mode 100644
index 000..78b7855
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -0,0 +1,225 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/maven-v4_0_0.xsd;>
+  4.0.0
+  
+org.apache.hadoop
+hadoop-yarn-applications
+3.0.0-alpha2-SNAPSHOT
+  
+  org.apache.hadoop
+  hadoop-yarn-services-api
+  Apache Hadoop YARN Services API
+  3.0.0-alpha2-SNAPSHOT
+  

[60/74] [abbrv] hadoop git commit: YARN-5813. Slider should not try to set a negative lifetime timeout value. Contributed by Jian He

[jianhe]

YARN-5813. Slider should not try to set a negative lifetime timeout value. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb8bd398
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb8bd398
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb8bd398

Branch: refs/heads/yarn-native-services
Commit: eb8bd39846c72a991940bdb5dd2e88dd690948e9
Parents: 4a3f1a4
Author: Gour Saha 
Authored: Tue Nov 1 17:39:54 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../src/main/java/org/apache/slider/client/SliderClient.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb8bd398/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d1f88c5..ea10ed0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2120,7 +2120,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 amLauncher.setKeepContainersOverRestarts(true);
 // set lifetime in submission context;
 Map appTimeout = new HashMap<>();
-appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+if (lifetime >= 0) {
+  appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+}
 amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
 int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
 amLauncher.setMaxAppAttempts(maxAppAttempts);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[42/74] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

YARN-5961. Generate native services protobuf classes during build. Contributed 
by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be43905b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be43905b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be43905b

Branch: refs/heads/yarn-native-services
Commit: be43905bdb08e5a667b228f26cebc2c5f14bcbc7
Parents: 8defcbb
Author: Jian He 
Authored: Sat Dec 3 13:27:04 2016 -0800
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../hadoop-yarn-slider-core/pom.xml |65 +-
 .../org/apache/slider/api/proto/Messages.java   | 34473 -
 .../slider/api/proto/SliderClusterAPI.java  |  2293 --
 3 files changed, 27 insertions(+), 36804 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be43905b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 10cf6b1..7453d12 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -37,6 +37,33 @@
 
 
   
+org.apache.hadoop
+hadoop-maven-plugins
+
+  
+compile-protoc
+
+  protoc
+
+
+  ${protobuf.version}
+  ${protoc.path}
+  
+${basedir}/src/main/proto
+  
+  
+${basedir}/src/main/proto
+
+  SliderClusterMessages.proto
+  SliderClusterProtocol.proto
+
+  
+
+  
+
+  
+
+  
 org.apache.maven.plugins
 maven-jar-plugin
 
@@ -85,9 +112,6 @@
   
 **/*.json
 src/main/resources/webapps/slideram/.keep
-
-
src/main/java/org/apache/slider/api/proto/Messages.java
-
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
   
 
   
@@ -362,41 +386,6 @@
 
   
 
-
-  compile-protobuf
-  
-
-  
-org.apache.hadoop
-hadoop-maven-plugins
-
-  
-compile-protoc
-generate-sources
-
-  protoc
-
-
-  ${protobuf.version}
-  protoc
-  
-${basedir}/src/main/proto
-  
-  
-${basedir}/src/main/proto
-
-  SliderClusterMessages.proto
-  SliderClusterProtocol.proto
-
-  
-  ${basedir}/src/main/java
-
-  
-
-  
-
-  
-
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[33/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
new file mode 100644
index 000..9879d05
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Serialized information about liveness
+ * 
+ *   If true liveness probes are implemented, this
+ *   datatype can be extended to publish their details.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ApplicationLivenessInformation {
+  /** flag set if the cluster is at size */
+  public boolean allRequestsSatisfied;
+
+  /** number of outstanding requests: those needed to satisfy */
+  public int requestsOutstanding;
+
+  /** number of requests submitted to YARN */
+  public int activeRequests;
+
+  @Override
+  public String toString() {
+final StringBuilder sb =
+new StringBuilder("ApplicationLivenessInformation{");
+sb.append("allRequestsSatisfied=").append(allRequestsSatisfied);
+sb.append(", requestsOutstanding=").append(requestsOutstanding);
+sb.append('}');
+return sb.toString();
+  }
+}
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
new file mode 100644
index 000..c46a59f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.server.appmaster.state.RoleStatus;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Serializable version of component data.
+ * 
+ * This is sent in REST 

[40/74] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be43905b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
deleted file mode 100644
index 081b7fa..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+++ /dev/null
@@ -1,2293 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: SliderClusterProtocol.proto
-
-package org.apache.slider.api.proto;
-
-public final class SliderClusterAPI {
-  private SliderClusterAPI() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  /**
-   * Protobuf service {@code org.apache.slider.api.SliderClusterProtocolPB}
-   *
-   * 
-   **
-   * Protocol used from between Slider Client and AM
-   * 
-   */
-  public static abstract class SliderClusterProtocolPB
-  implements com.google.protobuf.Service {
-protected SliderClusterProtocolPB() {}
-
-public interface Interface {
-  /**
-   * rpc stopCluster(.org.apache.slider.api.StopClusterRequestProto) 
returns (.org.apache.slider.api.StopClusterResponseProto);
-   */
-  public abstract void stopCluster(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
upgradeContainers(.org.apache.slider.api.UpgradeContainersRequestProto) returns 
(.org.apache.slider.api.UpgradeContainersResponseProto);
-   *
-   * 
-   **
-   * Upgrade containers 
-   * 
-   */
-  public abstract void upgradeContainers(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto 
request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc flexCluster(.org.apache.slider.api.FlexClusterRequestProto) 
returns (.org.apache.slider.api.FlexClusterResponseProto);
-   *
-   * 
-   **
-   * Flex the cluster. 
-   * 
-   */
-  public abstract void flexCluster(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
getJSONClusterStatus(.org.apache.slider.api.GetJSONClusterStatusRequestProto) 
returns (.org.apache.slider.api.GetJSONClusterStatusResponseProto);
-   *
-   * 
-   **
-   * Get the current cluster status
-   * 
-   */
-  public abstract void getJSONClusterStatus(
-  com.google.protobuf.RpcController controller,
-  
org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
getInstanceDefinition(.org.apache.slider.api.GetInstanceDefinitionRequestProto) 
returns (.org.apache.slider.api.GetInstanceDefinitionResponseProto);
-   *
-   * 
-   **
-   * Get the instance definition
-   * 
-   */
-  public abstract void getInstanceDefinition(
-  com.google.protobuf.RpcController controller,
-  
org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
listNodeUUIDsByRole(.org.apache.slider.api.ListNodeUUIDsByRoleRequestProto) 
returns (.org.apache.slider.api.ListNodeUUIDsByRoleResponseProto);
-   *
-   * 
-   **
-   * List all running nodes in a role
-   * 
-   */
-  public abstract void listNodeUUIDsByRole(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto 
request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc getNode(.org.apache.slider.api.GetNodeRequestProto) returns 
(.org.apache.slider.api.GetNodeResponseProto);
-   *
-   * 
-   **
-   * Get the details on a node
-   * 
-   */
-  public abstract void getNode(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 

[34/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
new file mode 100644
index 000..081b7fa
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
@@ -0,0 +1,2293 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: SliderClusterProtocol.proto
+
+package org.apache.slider.api.proto;
+
+public final class SliderClusterAPI {
+  private SliderClusterAPI() {}
+  public static void registerAllExtensions(
+  com.google.protobuf.ExtensionRegistry registry) {
+  }
+  /**
+   * Protobuf service {@code org.apache.slider.api.SliderClusterProtocolPB}
+   *
+   * 
+   **
+   * Protocol used from between Slider Client and AM
+   * 
+   */
+  public static abstract class SliderClusterProtocolPB
+  implements com.google.protobuf.Service {
+protected SliderClusterProtocolPB() {}
+
+public interface Interface {
+  /**
+   * rpc stopCluster(.org.apache.slider.api.StopClusterRequestProto) 
returns (.org.apache.slider.api.StopClusterResponseProto);
+   */
+  public abstract void stopCluster(
+  com.google.protobuf.RpcController controller,
+  org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc 
upgradeContainers(.org.apache.slider.api.UpgradeContainersRequestProto) returns 
(.org.apache.slider.api.UpgradeContainersResponseProto);
+   *
+   * 
+   **
+   * Upgrade containers 
+   * 
+   */
+  public abstract void upgradeContainers(
+  com.google.protobuf.RpcController controller,
+  org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto 
request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc flexCluster(.org.apache.slider.api.FlexClusterRequestProto) 
returns (.org.apache.slider.api.FlexClusterResponseProto);
+   *
+   * 
+   **
+   * Flex the cluster. 
+   * 
+   */
+  public abstract void flexCluster(
+  com.google.protobuf.RpcController controller,
+  org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc 
getJSONClusterStatus(.org.apache.slider.api.GetJSONClusterStatusRequestProto) 
returns (.org.apache.slider.api.GetJSONClusterStatusResponseProto);
+   *
+   * 
+   **
+   * Get the current cluster status
+   * 
+   */
+  public abstract void getJSONClusterStatus(
+  com.google.protobuf.RpcController controller,
+  
org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc 
getInstanceDefinition(.org.apache.slider.api.GetInstanceDefinitionRequestProto) 
returns (.org.apache.slider.api.GetInstanceDefinitionResponseProto);
+   *
+   * 
+   **
+   * Get the instance definition
+   * 
+   */
+  public abstract void getInstanceDefinition(
+  com.google.protobuf.RpcController controller,
+  
org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc 
listNodeUUIDsByRole(.org.apache.slider.api.ListNodeUUIDsByRoleRequestProto) 
returns (.org.apache.slider.api.ListNodeUUIDsByRoleResponseProto);
+   *
+   * 
+   **
+   * List all running nodes in a role
+   * 
+   */
+  public abstract void listNodeUUIDsByRole(
+  com.google.protobuf.RpcController controller,
+  org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto 
request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc getNode(.org.apache.slider.api.GetNodeRequestProto) returns 
(.org.apache.slider.api.GetNodeResponseProto);
+   *
+   * 
+   **
+   * Get the details on a node
+   * 
+   */
+  public abstract void getNode(
+  com.google.protobuf.RpcController controller,
+  org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
+  
com.google.protobuf.RpcCallback
 done);
+
+  /**
+   * rpc 

[36/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
new file mode 100644
index 000..373d64d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -0,0 +1,34473 @@
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: SliderClusterMessages.proto
+
+package org.apache.slider.api.proto;
+
+public final class Messages {
+  private Messages() {}
+  public static void registerAllExtensions(
+  com.google.protobuf.ExtensionRegistry registry) {
+  }
+  public interface RoleInstanceStateOrBuilder
+  extends com.google.protobuf.MessageOrBuilder {
+
+// required string name = 1;
+/**
+ * required string name = 1;
+ */
+boolean hasName();
+/**
+ * required string name = 1;
+ */
+java.lang.String getName();
+/**
+ * required string name = 1;
+ */
+com.google.protobuf.ByteString
+getNameBytes();
+
+// optional string role = 2;
+/**
+ * optional string role = 2;
+ */
+boolean hasRole();
+/**
+ * optional string role = 2;
+ */
+java.lang.String getRole();
+/**
+ * optional string role = 2;
+ */
+com.google.protobuf.ByteString
+getRoleBytes();
+
+// required uint32 state = 4;
+/**
+ * required uint32 state = 4;
+ */
+boolean hasState();
+/**
+ * required uint32 state = 4;
+ */
+int getState();
+
+// required uint32 exitCode = 5;
+/**
+ * required uint32 exitCode = 5;
+ */
+boolean hasExitCode();
+/**
+ * required uint32 exitCode = 5;
+ */
+int getExitCode();
+
+// optional string command = 6;
+/**
+ * optional string command = 6;
+ */
+boolean hasCommand();
+/**
+ * optional string command = 6;
+ */
+java.lang.String getCommand();
+/**
+ * optional string command = 6;
+ */
+com.google.protobuf.ByteString
+getCommandBytes();
+
+// optional string diagnostics = 7;
+/**
+ * optional string diagnostics = 7;
+ */
+boolean hasDiagnostics();
+/**
+ * optional string diagnostics = 7;
+ */
+java.lang.String getDiagnostics();
+/**
+ * optional string diagnostics = 7;
+ */
+com.google.protobuf.ByteString
+getDiagnosticsBytes();
+
+// repeated string output = 8;
+/**
+ * repeated string output = 8;
+ */
+java.util.List
+getOutputList();
+/**
+ * repeated string output = 8;
+ */
+int getOutputCount();
+/**
+ * repeated string output = 8;
+ */
+java.lang.String getOutput(int index);
+/**
+ * repeated string output = 8;
+ */
+com.google.protobuf.ByteString
+getOutputBytes(int index);
+
+// repeated string environment = 9;
+/**
+ * repeated string environment = 9;
+ */
+java.util.List
+getEnvironmentList();
+/**
+ * repeated string environment = 9;
+ */
+int getEnvironmentCount();
+/**
+ * repeated string environment = 9;
+ */
+java.lang.String getEnvironment(int index);
+/**
+ * repeated string environment = 9;
+ */
+com.google.protobuf.ByteString
+getEnvironmentBytes(int index);
+
+// required uint32 roleId = 10;
+/**
+ * required uint32 roleId = 10;
+ */
+boolean hasRoleId();
+/**
+ * required uint32 roleId = 10;
+ */
+int getRoleId();
+
+// required bool released = 11;
+/**
+ * required bool released = 11;
+ */
+boolean hasReleased();
+/**
+ * required bool released = 11;
+ */
+boolean getReleased();
+
+// required int64 createTime = 12;
+/**
+ * required int64 createTime = 12;
+ */
+boolean hasCreateTime();
+/**
+ * required int64 createTime = 12;
+ */
+long getCreateTime();
+
+// required int64 startTime = 13;
+/**
+ * required int64 startTime = 13;
+ */
+boolean hasStartTime();
+/**
+ * required int64 startTime = 13;
+ */
+long getStartTime();
+
+// required string host = 14;
+/**
+ * required string host = 14;
+ */
+boolean hasHost();
+/**
+ * required string host = 14;
+ */
+java.lang.String getHost();
+/**
+ * required string host = 14;
+ */
+

[57/74] [abbrv] hadoop git commit: YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22be358/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
deleted file mode 100644
index e82ad84..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-//import java.nio.file.Files;
-//import java.nio.file.Path;
-//import java.nio.file.Paths;
-//import java.nio.file.attribute.PosixFilePermission;
-//import java.nio.file.attribute.PosixFilePermissions;
-
-
-/**
- *
- */
-public class SecurityUtils {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(SecurityUtils.class);
-
-  private static String CA_CONFIG_CONTENTS =  "HOME= .\n"
-+ "RANDFILE= 
$ENV::HOME/.rnd\n\n"
-+ "[ ca ]\n"
-+ "default_ca = 
CA_CLIENT\n"
-+ "[ CA_CLIENT ]\n"
-+ "dir= 
${SEC_DIR}/db\n"
-+ "certs  = 
$dir/certs\n"
-+ "new_certs_dir  = 
$dir/newcerts\n"
-+ "\n"
-+ "database   = 
$dir/index.txt\n"
-+ "serial = 
$dir/serial\n"
-+ "default_days   = 365
\n"
-+ "\n"
-+ "default_crl_days   = 7  \n"
-+ "default_md = sha256 
\n"
-+ "\n"
-+ "policy = 
policy_anything \n"
-+ "\n"
-+ "[ policy_anything ]\n"
-+ "countryName= 
optional\n"
-+ "stateOrProvinceName= 
optional\n"
-+ "localityName   = 
optional\n"
-+ "organizationName   = 
optional\n"
-+ "organizationalUnitName = 
optional\n"
-+ "commonName = 
optional\n"
-+ "emailAddress   = 
optional\n"
-+ "\n"
-+ "[req]\n"
-+ "distinguished_name = 
req_distinguished_name\n"
-+ 

[29/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
new file mode 100644
index 000..739b5fc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = {SliderActions.ACTION_LIST},
+commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
+
+public class ActionListArgs extends AbstractActionArgs {
+  @Override
+  public String getActionName() {
+return SliderActions.ACTION_LIST;
+  }
+
+  @Parameter(names = {ARG_LIVE},
+  description = "List only live application instances")
+  public boolean live;
+
+  @Parameter(names = {ARG_STATE},
+  description = "list only applications in the specific YARN state")
+  public String state = "";
+  
+  @Parameter(names = {ARG_VERBOSE},
+  description = "print out information in details")
+  public boolean verbose = false;
+
+  @Parameter(names = {ARG_CONTAINERS},
+  description = "List containers of an application instance")
+  public boolean containers;
+
+  @Parameter(names = {ARG_VERSION},
+  description = "Filter containers by app version (used with " +
+ARG_CONTAINERS + ")")
+  public String version;
+
+  @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
+  description = "Filter containers by component names (used with " +
+ARG_CONTAINERS + ")")
+  public Set components = new HashSet<>(0);
+
+  /**
+   * Get the min #of params expected
+   * @return the min number of params in the {@link #parameters} field
+   */
+  public int getMinParams() {
+return 0;
+  }
+
+  @Override
+  public int getMaxParams() {
+return 1;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
new file mode 100644
index 000..1b73522
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the 

[28/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
new file mode 100644
index 000..5140059
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class ComponentArgsDelegate extends AbstractArgsDelegate {
+
+  /**
+   * This is a listing of the roles to create
+   */
+  @Parameter(names = {ARG_COMPONENT,  ARG_COMPONENT_SHORT, ARG_ROLE},
+ arity = 2,
+ description = "--component   e.g. +1 incr by 1, -2 
decr by 2, and 3 makes final count 3",
+ splitter = DontSplitArguments.class)
+  public List componentTuples = new ArrayList<>(0);
+
+
+  /**
+   * Get the role mapping (may be empty, but never null)
+   * @return role mapping
+   * @throws BadCommandArgumentsException parse problem
+   */
+  public Map getComponentMap() throws 
BadCommandArgumentsException {
+return convertTupleListToMap("component", componentTuples);
+  }
+
+  public List getComponentTuples() {
+return componentTuples;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
new file mode 100644
index 000..0344305
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.converters.IParameterSplitter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DontSplitArguments implements IParameterSplitter {
+
+  @Override
+  public List split(String value) {
+List list = new ArrayList<>(1);
+list.add(value);
+return list;
+  }
+}

[27/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
new file mode 100644
index 000..aa5edf1
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -0,0 +1,915 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
+import org.apache.slider.core.persist.Filenames;
+import org.apache.slider.core.persist.InstancePaths;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+import static 
org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS;
+import static 
org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS;
+
+public class CoreFileSystem {
+  private static final Logger
+log = LoggerFactory.getLogger(CoreFileSystem.class);
+
+  private static final String UTF_8 = "UTF-8";
+
+  protected final FileSystem fileSystem;
+  protected final Configuration configuration;
+
+  public CoreFileSystem(FileSystem fileSystem, Configuration configuration) {
+Preconditions.checkNotNull(fileSystem,
+   "Cannot create a CoreFileSystem with a null 
FileSystem");
+Preconditions.checkNotNull(configuration,
+   "Cannot create a CoreFileSystem with a null 
Configuration");
+this.fileSystem = fileSystem;
+this.configuration = configuration;
+  }
+
+  public CoreFileSystem(Configuration configuration) throws IOException {
+Preconditions.checkNotNull(configuration,
+   "Cannot create a CoreFileSystem with a null 
Configuration");
+this.fileSystem = FileSystem.get(configuration);
+this.configuration = fileSystem.getConf();
+  }
+  
+  /**
+   * Get the temp path for this cluster
+   * @param clustername name of the cluster
+   * @return path for temp files (is not purged)
+   */
+  public Path getTempPathForCluster(String clustername) {
+Path clusterDir = buildClusterDirPath(clustername);
+return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX);
+  }
+
+  /**
+   * Returns the 

[17/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
new file mode 100644
index 000..78bb8c1
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Component defined in master package metainfo.json
+ */
+public class Component extends AbstractComponent {
+
+  String category = CATEGORY_MASTER;
+  String publishConfig = Boolean.FALSE.toString();
+  String minInstanceCount = "0";
+  String maxInstanceCount;
+  String autoStartOnFailure = Boolean.FALSE.toString();
+  String appExports;
+  String compExports;
+  String type = TYPE_STANDARD;
+  List componentExports = new ArrayList<>();
+  List dockerContainers = new ArrayList<>();
+  List configFiles = new ArrayList<>();
+
+  public Component() {
+  }
+
+  public String getType() {
+return type;
+  }
+
+  public void setType(String type) {
+this.type = type;
+  }
+
+  public String getCategory() {
+return category;
+  }
+
+  public void setCategory(String category) {
+this.category = category;
+  }
+
+  public String getPublishConfig() {
+return publishConfig;
+  }
+
+  public void setPublishConfig(String publishConfig) {
+this.publishConfig = publishConfig;
+  }
+
+  public String getAutoStartOnFailure() {
+return autoStartOnFailure;
+  }
+
+  public void setAutoStartOnFailure(String autoStartOnFailure) {
+this.autoStartOnFailure = autoStartOnFailure;
+  }
+
+  public String getAppExports() {
+return appExports;
+  }
+
+  public void setAppExports(String appExports) {
+this.appExports = appExports;
+  }
+
+  public String getCompExports() {
+return compExports;
+  }
+
+  public void setCompExports(String compExports) {
+this.compExports = compExports;
+  }
+
+  public String getMinInstanceCount() {
+return minInstanceCount;
+  }
+  
+  @JsonProperty("dockerContainers")
+  public List getDockerContainers() {
+ return this.dockerContainers;
+  }
+  
+  public Boolean getAutoStartOnFailureBoolean() {
+if (SliderUtils.isUnset(getAutoStartOnFailure())) {
+  return Boolean.FALSE;
+}
+
+return Boolean.parseBoolean(getAutoStartOnFailure());
+  }
+
+  public int getMinInstanceCountInt() throws BadConfigException {
+if (SliderUtils.isUnset(minInstanceCount)) {
+  return 0;
+}
+
+try {
+  return Integer.parseInt(minInstanceCount);
+} catch (NumberFormatException nfe) {
+  throw new BadConfigException(nfe, "Invalid value for minInstanceCount 
for %s", name);
+}
+  }
+
+  public int getMaxInstanceCountInt() throws BadConfigException {
+if (SliderUtils.isUnset(maxInstanceCount)) {
+  return Integer.MAX_VALUE;
+}
+
+try {
+  return Integer.parseInt(maxInstanceCount);
+} catch (NumberFormatException nfe) {
+  throw new BadConfigException(nfe, "Invalid value for maxInstanceCount 
for %s", name);
+}
+  }
+
+  public void setMinInstanceCount(String minInstanceCount) {
+this.minInstanceCount = minInstanceCount;
+  }
+
+  public String getMaxInstanceCount() {
+return 

[07/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
new file mode 100644
index 000..781ae00
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentEnv.java
@@ -0,0 +1,376 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import com.google.gson.annotations.SerializedName;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class AgentEnv {
+
+  /**
+   * Various directories, configurable in ambari-agent.ini
+   */
+  private Directory[] stackFoldersAndFiles = new Directory[0];
+
+  /**
+   * Directories that match name /etc/alternatives/*conf
+   */
+  private Alternative[] alternatives = new Alternative[0];
+
+  /**
+   * List of existing users
+   */
+  private ExistingUser[] existingUsers = new ExistingUser[0];
+
+  /**
+   * List of repos
+   */
+  private String[] existingRepos = new String[0];
+
+  /**
+   * List of packages
+   */
+  private PackageDetail[] installedPackages = new PackageDetail[0];
+
+  /**
+   * The host health report
+   */
+  private HostHealth hostHealth = new HostHealth();
+
+  private Integer umask;
+
+  private Boolean iptablesIsRunning;
+
+  public Integer getUmask() {
+return umask;
+  }
+
+  public void setUmask(Integer umask) {
+this.umask = umask;
+  }
+
+  public Directory[] getStackFoldersAndFiles() {
+return stackFoldersAndFiles;
+  }
+
+  public void setStackFoldersAndFiles(Directory[] dirs) {
+stackFoldersAndFiles = dirs;
+  }
+
+  public void setExistingUsers(ExistingUser[] users) {
+existingUsers = users;
+  }
+
+  public ExistingUser[] getExistingUsers() {
+return existingUsers;
+  }
+
+  public void setAlternatives(Alternative[] dirs) {
+alternatives = dirs;
+  }
+
+  public Alternative[] getAlternatives() {
+return alternatives;
+  }
+
+  public void setExistingRepos(String[] repos) {
+existingRepos = repos;
+  }
+
+  public String[] getExistingRepos() {
+return existingRepos;
+  }
+
+  public void setInstalledPackages(PackageDetail[] packages) {
+installedPackages = packages;
+  }
+
+  public PackageDetail[] getInstalledPackages() {
+return installedPackages;
+  }
+
+  public void setHostHealth(HostHealth healthReport) {
+hostHealth = healthReport;
+  }
+
+  public HostHealth getHostHealth() {
+return hostHealth;
+  }
+
+  public Boolean getIptablesIsRunning() {
+return iptablesIsRunning;
+  }
+
+  public void setIptablesIsRunning(Boolean iptablesIsRunning) {
+this.iptablesIsRunning = iptablesIsRunning;
+  }
+
+  public static class HostHealth {
+/**
+ * Java processes running on the system.  Default empty array.
+ */
+@SerializedName("activeJavaProcs")
+private JavaProc[] activeJavaProcs = new JavaProc[0];
+
+/**
+ * The current time when agent send the host check report
+ */
+@SerializedName("agentTimeStampAtReporting")
+private long agentTimeStampAtReporting = 0;
+
+/**
+ * The current time when host check report was received
+ */
+@SerializedName("serverTimeStampAtReporting")
+private long serverTimeStampAtReporting = 0;
+
+/**
+ * Live services running on the agent
+ */
+@SerializedName("liveServices")
+private LiveService[] liveServices = new LiveService[0];
+
+   

[46/74] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/689cbc40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
new file mode 100644
index 000..cfcae95
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.slider.providers.PlacementPolicy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+/**
+ * An Application resource has the following attributes.
+ **/
+
+@ApiModel(description = "An Application resource has the following 
attributes.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, 
containers " })
+public class Application extends BaseResource {
+  private static final long serialVersionUID = -4491694636566094885L;
+
+  private String id = null;
+  private String name = null;
+  private Artifact artifact = null;
+  private Resource resource = null;
+  private String launchCommand = null;
+  private Date launchTime = null;
+  private Long numberOfContainers = null;
+  private Long numberOfRunningContainers = null;
+  private Long lifetime = null;
+  private PlacementPolicy placementPolicy = null;
+  private List components = null;
+  private Configuration configuration = null;
+  private List containers = new ArrayList<>();
+  private ApplicationState state = null;
+  private Map quicklinks = null;
+  private String queue;
+
+  /**
+   * A unique application id.
+   **/
+  public Application id(String id) {
+this.id = id;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", required = true, value = "A unique 
application id.")
+  @JsonProperty("id")
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  /**
+   * A unique application name.
+   **/
+  public Application name(String name) {
+this.name = name;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", required = true, value = "A unique 
application name.")
+  @JsonProperty("name")
+  public String getName() {
+return name;
+  }
+
+  public void setName(String name) {
+this.name = name;
+  }
+
+  /**
+   * Artifact of single-component applications. Mandatory if components
+   * attribute is not specified.
+   **/
+  public Application artifact(Artifact artifact) {
+this.artifact = artifact;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Artifact of single-component 
applications. Mandatory if components attribute is not specified.")
+  @JsonProperty("artifact")
+  public Artifact getArtifact() {
+return artifact;
+  }
+
+  public void setArtifact(Artifact artifact) {
+this.artifact = artifact;
+  }
+
+  /**
+   * Resource of single-component applications or the global default for
+   * multi-component applications. Mandatory if it is a single-component
+   * application and if cpus and memory are not specified at the 

[31/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
new file mode 100644
index 000..30f6ba9
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.client;
+
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.types.NodeInformationList;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionClientArgs;
+import org.apache.slider.common.params.ActionDependencyArgs;
+import org.apache.slider.common.params.ActionDestroyArgs;
+import org.apache.slider.common.params.ActionDiagnosticArgs;
+import org.apache.slider.common.params.ActionEchoArgs;
+import org.apache.slider.common.params.ActionFlexArgs;
+import org.apache.slider.common.params.ActionFreezeArgs;
+import org.apache.slider.common.params.ActionInstallKeytabArgs;
+import org.apache.slider.common.params.ActionInstallPackageArgs;
+import org.apache.slider.common.params.ActionKeytabArgs;
+import org.apache.slider.common.params.ActionNodesArgs;
+import org.apache.slider.common.params.ActionPackageArgs;
+import org.apache.slider.common.params.ActionKillContainerArgs;
+import org.apache.slider.common.params.ActionListArgs;
+import org.apache.slider.common.params.ActionRegistryArgs;
+import org.apache.slider.common.params.ActionResolveArgs;
+import org.apache.slider.common.params.ActionResourceArgs;
+import org.apache.slider.common.params.ActionStatusArgs;
+import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionUpgradeArgs;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.providers.AbstractClientProvider;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Interface of those method calls in the slider API that are intended
+ * for direct public invocation.
+ * 
+ * Stability: evolving
+ */
+public interface SliderClientAPI extends Service {
+  /**
+   * Destroy a cluster. There's two race conditions here
+   * #1 the cluster is started between verifying that there are no live
+   * clusters of that name.
+   */
+  int actionDestroy(String clustername, ActionDestroyArgs destroyArgs)
+  throws YarnException, IOException;
+
+  int actionDestroy(String clustername) throws YarnException,
+  IOException;
+
+  /**
+   * AM to commit an asynchronous suicide
+   */
+  int actionAmSuicide(String clustername,
+  ActionAMSuicideArgs args) throws YarnException, IOException;
+
+  /**
+   * Get the provider for this cluster
+   * @param provider the name of the provider
+   * @return the provider instance
+   * @throws SliderException problems building the provider
+   */
+  AbstractClientProvider createClientProvider(String provider)
+throws SliderException;
+
+  /**
+   * Build up the cluster specification/directory
+   *
+   * @param clustername cluster name
+   * @param buildInfo the arguments needed to build the cluster
+   * @throws YarnException Yarn problems
+   * @throws IOException other 

[39/74] [abbrv] hadoop git commit: Addendum patch for YARN-5610. Contributed by Gour Saha

[jianhe]

Addendum patch for YARN-5610. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f89aa15e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f89aa15e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f89aa15e

Branch: refs/heads/yarn-native-services
Commit: f89aa15eb78aa5e15692830db820983d05e2a0fa
Parents: 3806344
Author: Jian He 
Authored: Wed Oct 12 13:33:09 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../yarn/services/resource/Application.java | 44 ++--
 .../services/resource/ApplicationState.java |  5 +++
 .../services/resource/ApplicationStatus.java|  8 ++--
 .../hadoop/yarn/services/resource/Artifact.java |  4 +-
 .../yarn/services/resource/Component.java   | 16 +++
 .../yarn/services/resource/Container.java   | 15 ---
 .../yarn/services/resource/ReadinessCheck.java  |  6 +--
 7 files changed, 54 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f89aa15e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index cfcae95..719bf95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -48,8 +48,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 public class Application extends BaseResource {
   private static final long serialVersionUID = -4491694636566094885L;
 
-  private String id = null;
   private String name = null;
+  private String id = null;
   private Artifact artifact = null;
   private Resource resource = null;
   private String launchCommand = null;
@@ -63,25 +63,7 @@ public class Application extends BaseResource {
   private List containers = new ArrayList<>();
   private ApplicationState state = null;
   private Map quicklinks = null;
-  private String queue;
-
-  /**
-   * A unique application id.
-   **/
-  public Application id(String id) {
-this.id = id;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", required = true, value = "A unique 
application id.")
-  @JsonProperty("id")
-  public String getId() {
-return id;
-  }
-
-  public void setId(String id) {
-this.id = id;
-  }
+  private String queue = null;
 
   /**
* A unique application name.
@@ -102,6 +84,24 @@ public class Application extends BaseResource {
   }
 
   /**
+   * A unique application id.
+   **/
+  public Application id(String id) {
+this.id = id;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "A unique application id.")
+  @JsonProperty("id")
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  /**
* Artifact of single-component applications. Mandatory if components
* attribute is not specified.
**/
@@ -423,8 +423,8 @@ public class Application extends BaseResource {
 sb.append("numberOfRunningContainers: ")
 .append(toIndentedString(numberOfRunningContainers)).append("\n");
 sb.append("lifetime: 
").append(toIndentedString(lifetime)).append("\n");
-sb.append("placementPolicy: ")
-.append(toIndentedString(placementPolicy)).append("\n");
+sb.append("placementPolicy: 
").append(toIndentedString(placementPolicy))
+.append("\n");
 sb.append("components: ").append(toIndentedString(components))
 .append("\n");
 sb.append("configuration: ").append(toIndentedString(configuration))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f89aa15e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
 

[37/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
new file mode 100644
index 000..f8e5e7c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -0,0 +1,795 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.api;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.SliderProviderFactory;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.annotate.JsonIgnore;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_HOME;
+import static org.apache.slider.api.OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
+import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+
+/**
+ * Represents a cluster specification; designed to be sendable over the wire
+ * and persisted in JSON by way of Jackson.
+ * 
+ * When used in cluster status operations the info
+ * and statistics maps contain information about the cluster.
+ * 
+ * As a wire format it is less efficient in both xfer and ser/deser than 
+ * a binary format, but by having one unified format for wire and persistence,
+ * the code paths are simplified.
+ *
+ * This was the original single-file specification/model used in the Hoya
+ * precursor to Slider. Its now retained primarily as a way to publish
+ * the current state of the application, or at least a fraction thereof ...
+ * the larger set of information from the REST API is beyond the scope of
+ * this structure.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+
+public class ClusterDescription implements Cloneable {
+  protected static final Logger
+log = LoggerFactory.getLogger(ClusterDescription.class);
+
+  private static final String UTF_8 = "UTF-8";
+
+  /**
+   * version counter
+   */
+  public String version = "1.0";
+
+  /**
+   * Name of the cluster
+   */
+  public String name;
+
+  /**
+   * Type of cluster
+   */
+  public String type = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
+
+  /**
+   * State of the cluster
+   */
+  public int state;
+  
+  /*
+   State list for both clusters and nodes in them. Ordered so that destroyed 
follows
+   stopped.
+   
+   Some of the states are only used for recording
+   the persistent state of the cluster and are not
+   seen in node descriptions
+   */
+
+  /**
+   * Specification is incomplete & 

[44/74] [abbrv] hadoop git commit: YARN-5770. Performance improvement of native-services REST API service. Contributed by Gour Saha

[jianhe]

YARN-5770. Performance improvement of native-services REST API service. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05c15d71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05c15d71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05c15d71

Branch: refs/heads/yarn-native-services
Commit: 05c15d71d1bfeec01f57ff0ccd268ffe154a72fd
Parents: 7b10d4c
Author: Billie Rinaldi 
Authored: Wed Oct 26 08:34:39 2016 -0700
Committer: Jian He 
Committed: Tue Dec 13 14:46:44 2016 -0800

--
 .../api/impl/ApplicationApiService.java | 144 +--
 .../yarn/services/resource/Application.java |   7 +-
 .../yarn/services/resource/Container.java   |   4 +-
 .../services/webapp/ApplicationApiWebApp.java   |  12 +-
 .../org/apache/slider/client/SliderClient.java  |  25 ++--
 .../apache/slider/client/SliderClientAPI.java   |  11 ++
 6 files changed, 107 insertions(+), 96 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/05c15d71/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 73df4a1..cf43ac2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -20,8 +20,7 @@ package org.apache.hadoop.yarn.services.api.impl;
 import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
 import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
 
-import java.io.File;
-import java.io.FileReader;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.security.PrivilegedExceptionAction;
@@ -36,7 +35,6 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.regex.Pattern;
 
-import javax.inject.Singleton;
 import javax.ws.rs.Consumes;
 import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
@@ -52,6 +50,7 @@ import javax.ws.rs.core.Response.Status;
 
 import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.PathNotFoundException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -78,7 +77,6 @@ import org.apache.slider.common.params.ActionFlexArgs;
 import org.apache.slider.common.params.ActionFreezeArgs;
 import org.apache.slider.common.params.ActionListArgs;
 import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
 import org.apache.slider.common.params.ActionThawArgs;
 import org.apache.slider.common.params.ComponentArgsDelegate;
 import org.apache.slider.common.tools.SliderUtils;
@@ -98,6 +96,7 @@ import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
 import com.google.gson.JsonParser;
+import com.google.inject.Singleton;
 
 @Singleton
 @Path(APPLICATIONS_API_RESOURCE_PATH)
@@ -109,6 +108,11 @@ public class ApplicationApiService implements 
ApplicationApi {
   private static org.apache.hadoop.conf.Configuration SLIDER_CONFIG;
   private static UserGroupInformation SLIDER_USER;
   private static SliderClient SLIDER_CLIENT;
+  private static Response SLIDER_VERSION;
+  private static final JsonParser JSON_PARSER = new JsonParser();
+  private static final JsonObject EMPTY_JSON_OBJECT = new JsonObject();
+  private static final ActionListArgs ACTION_LIST_ARGS = new ActionListArgs();
+  private static final ActionFreezeArgs ACTION_FREEZE_ARGS = new 
ActionFreezeArgs();
 
   static {
 init();
@@ -119,24 +123,27 @@ public class ApplicationApiService implements 
ApplicationApi {
 SLIDER_CONFIG = getSliderClientConfiguration();
 SLIDER_USER = getSliderUser();
 SLIDER_CLIENT = createSliderClient();
+SLIDER_VERSION = initSliderVersion();
   }
 
   @GET
-  @Path("/slider-version")
+  @Path("/versions/slider-version")
   @Consumes({ MediaType.APPLICATION_JSON })
   

[35/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
new file mode 100644
index 000..17fd965
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.proto;
+
+import com.google.protobuf.ByteString;
+import org.apache.commons.io.IOUtils;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeEntryInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.persist.AggregateConfSerDeser;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.server.services.security.SecurityStore;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to handle marshalling of REST
+ * types to/from Protobuf records.
+ */
+public class RestTypeMarshalling {
+
+  public static Messages.ApplicationLivenessInformationProto
+  marshall(ApplicationLivenessInformation info) {
+
+Messages.ApplicationLivenessInformationProto.Builder builder =
+Messages.ApplicationLivenessInformationProto.newBuilder();
+builder.setAllRequestsSatisfied(info.allRequestsSatisfied);
+builder.setRequestsOutstanding(info.requestsOutstanding);
+return builder.build();
+  }
+
+  public static ApplicationLivenessInformation
+  unmarshall(Messages.ApplicationLivenessInformationProto wire) {
+ApplicationLivenessInformation info = new ApplicationLivenessInformation();
+info.allRequestsSatisfied = wire.getAllRequestsSatisfied();
+info.requestsOutstanding = wire.getRequestsOutstanding();
+return info;
+  }
+
+  public static ComponentInformation
+  unmarshall(Messages.ComponentInformationProto wire) {
+ComponentInformation info = new ComponentInformation();
+info.name = wire.getName();
+info.priority = wire.getPriority();
+info.placementPolicy = wire.getPlacementPolicy();
+
+info.actual = wire.getActual();
+info.completed = wire.getCompleted();
+info.desired = wire.getDesired();
+info.failed = wire.getFailed();
+info.releasing = wire.getReleasing();
+info.requested = wire.getRequested();
+info.started = wire.getStarted();
+info.startFailed = wire.getStartFailed();
+info.totalRequested = wire.getTotalRequested();
+info.containers = new ArrayList<>(wire.getContainersList());
+if (wire.hasFailureMessage()) {
+  info.failureMessage = wire.getFailureMessage();
+}
+if (wire.hasPendingAntiAffineRequestCount()) {
+  info.pendingAntiAffineRequestCount = 
wire.getPendingAntiAffineRequestCount();
+}
+if (wire.hasIsAARequestOutstanding()) {
+  info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+}
+return info;
+  }
+
+  public static Messages.GetCertificateStoreResponseProto marshall(
+  SecurityStore securityStore) throws IOException {
+Messages.GetCertificateStoreResponseProto.Builder builder =
+

[24/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
new file mode 100644
index 000..5a3eb3d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -0,0 +1,528 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.MapOperations;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Launcher of applications: base class
+ */
+public abstract class AbstractLauncher extends Configured {
+  private static final Logger log =
+LoggerFactory.getLogger(AbstractLauncher.class);
+  public static final String CLASSPATH = "CLASSPATH";
+  /**
+   * Filesystem to use for the launch
+   */
+  protected final CoreFileSystem coreFileSystem;
+  /**
+   * Env vars; set up at final launch stage
+   */
+  protected final Map envVars = new HashMap<>();
+  protected final MapOperations env = new MapOperations("env", envVars);
+  protected final ContainerLaunchContext containerLaunchContext =
+Records.newRecord(ContainerLaunchContext.class);
+  protected final List commands = new ArrayList<>(20);
+  protected final Map localResources = new HashMap<>();
+  protected final Map mountPaths = new HashMap<>();
+  private final Map serviceData = new HashMap<>();
+  // security
+  protected final Credentials credentials;
+  protected LogAggregationContext logAggregationContext;
+  protected boolean yarnDockerMode = false;
+  protected String dockerImage;
+  protected String yarnContainerMountPoints;
+  protected String runPrivilegedContainer;
+
+
+  /**
+   * Create instance.
+   * @param conf configuration
+   * @param coreFileSystem filesystem
+   * @param credentials initial set of credentials -null is permitted
+   */
+  protected AbstractLauncher(Configuration conf,
+  CoreFileSystem coreFileSystem,
+  Credentials credentials) {
+super(conf);
+this.coreFileSystem = coreFileSystem;
+this.credentials = credentials != null ? credentials: new Credentials();
+  }
+
+  /**
+   * Get the container. Until "completed", this isn't valid to launch.
+   * @return the container to launch
+   */
+  public ContainerLaunchContext getContainerLaunchContext() {
+return 

[18/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
new file mode 100644
index 000..281895a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class AgentRoles {
+
+  /**
+   * List of roles Agent provider does not have any roles by default. All 
roles are read from the application
+   * specification.
+   */
+  protected static final List ROLES =
+  new ArrayList();
+
+  public static List getRoles() {
+return ROLES;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
new file mode 100644
index 000..cfcfc5d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import 
org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
+import 
org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
+import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
+import 
org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ *
+ */
+public class AgentUtils {
+  private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
+
+  

[26/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
new file mode 100644
index 000..73e0879
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -0,0 +1,2548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.slider.Slider;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.RoleKeys;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.params.Arguments;
+import org.apache.slider.common.params.SliderActions;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ClasspathConstructor;
+import org.apache.slider.core.main.LauncherExitCodes;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.services.utility.PatternValidator;
+import org.apache.slider.server.services.workflow.ForkedProcessService;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintWriter;
+import 

[25/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
new file mode 100644
index 000..86025ee
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.tools;
+
+import org.apache.hadoop.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.Locale;
+import java.util.Properties;
+
+/**
+ * Extract the version properties, which will look something like
+ * 
+ * application.name=${pom.name}
+ * application.version=${pom.version}
+ * application.build=${buildNumber}
+ * application.build.java.version=${java.version}
+ * application.build.info=${pom.name}-${pom.version} Built against 
${buildNumber} on ${java.version} by ${user.name}
+ * 
+ * 
+ * the mvn process-resources target will expand the properties
+ * and add the resources to target/classes, which will then look something like
+ * 
+ *   application.name=Slider Core
+ *   application.version=0.7.1-SNAPSHOT
+ *   application.build=1dd69
+ *   application.build.java.version=1.7.0_45
+ *   application.build.user=stevel
+ *   application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 
1.7.0_45 by stevel
+ * 
+ * 
+ * Note: the values will change and more properties added.
+ */
+public class SliderVersionInfo {
+  private static final Logger log = 
LoggerFactory.getLogger(SliderVersionInfo.class);
+
+  /**
+   * Name of the resource containing the filled-in-at-runtime props
+   */
+  public static final String VERSION_RESOURCE =
+  "org/apache/slider/providers/dynamic/application.properties";
+
+  public static final String APP_NAME = "application.name";
+  public static final String APP_VERSION = "application.version";
+  public static final String APP_BUILD = "application.build";
+  public static final String APP_BUILD_JAVA_VERSION = 
"application.build.java.version";
+  public static final String APP_BUILD_USER = "application.build.user";
+  public static final String APP_BUILD_INFO = "application.build.info";
+  public static final String HADOOP_BUILD_INFO = "hadoop.build.info";
+  public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info";
+
+
+  public static Properties loadVersionProperties()  {
+Properties props = new Properties();
+URL resURL = SliderVersionInfo.class.getClassLoader()
+   .getResource(VERSION_RESOURCE);
+assert resURL != null : "Null resource " + VERSION_RESOURCE;
+
+try {
+  InputStream inStream = resURL.openStream();
+  assert inStream != null : "Null input stream from " + VERSION_RESOURCE;
+  props.load(inStream);
+} catch (IOException e) {
+  log.warn("IOE loading " + VERSION_RESOURCE, e);
+}
+return props;
+  }
+
+  /**
+   * Load the version info and print it
+   * @param logger logger
+   */
+  public static void loadAndPrintVersionInfo(Logger logger) {
+Properties props = loadVersionProperties();
+logger.info(props.getProperty(APP_BUILD_INFO));
+logger.info("Compiled against Hadoop {}",
+props.getProperty(HADOOP_BUILD_INFO));
+logger.info(getHadoopVersionString());
+  }
+  
+  public static String getHadoopVersionString() {
+return String.format(Locale.ENGLISH,
+"Hadoop runtime version %s with source checksum %s and build date %s",
+VersionInfo.getBranch(),
+

[15/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
new file mode 100644
index 000..b767059
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -0,0 +1,2450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.health.HealthCheckRegistry;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
+import com.google.common.base.Preconditions;
+import com.google.protobuf.BlockingService;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.service.ServiceStateChangeListener;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
+import static org.apache.slider.common.Constants.HADOOP_JAAS_DEBUG;
+
+import 
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;

[30/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
new file mode 100644
index 000..5758f79
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderExitCodes.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.common;
+
+import org.apache.slider.core.main.LauncherExitCodes;
+
+public interface SliderExitCodes extends LauncherExitCodes {
+
+  /**
+   * starting point for exit codes; not an exception itself
+   */
+  int _EXIT_CODE_BASE =   64;
+
+  /**
+   * service entered the failed state: {@value}
+   */
+  int EXIT_YARN_SERVICE_FAILED =  65;
+
+  /**
+   * service was killed: {@value}
+   */
+  int EXIT_YARN_SERVICE_KILLED =  66;
+
+  /**
+   * timeout on monitoring client: {@value}
+   */
+  int EXIT_TIMED_OUT =67;
+
+  /**
+   * service finished with an error: {@value}
+   */
+  int EXIT_YARN_SERVICE_FINISHED_WITH_ERROR = 68;
+
+  /**
+   * the application instance is unknown: {@value}
+   */
+  int EXIT_UNKNOWN_INSTANCE = 69;
+
+  /**
+   * the application instance is in the wrong state for that operation: 
{@value}
+   */
+  int EXIT_BAD_STATE =70;
+
+  /**
+   * A spawned master process failed 
+   */
+  int EXIT_PROCESS_FAILED =   71;
+
+  /**
+   * The instance failed -too many containers were
+   * failing or some other threshold was reached
+   */
+  int EXIT_DEPLOYMENT_FAILED =72;
+
+  /**
+   * The application is live -and the requested operation
+   * does not work if the cluster is running
+   */
+  int EXIT_APPLICATION_IN_USE =   73;
+
+  /**
+   * There already is an application instance of that name
+   * when an attempt is made to create a new instance
+   */
+  int EXIT_INSTANCE_EXISTS =  75;
+
+  /**
+   * Exit code when the configurations in valid/incomplete: {@value}
+   */
+  int EXIT_BAD_CONFIGURATION =77;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
new file mode 100644
index 000..ba3effc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  

[32/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
new file mode 100644
index 000..8210f4d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -0,0 +1,4569 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.slider.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.io.Files;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.exceptions.NoRecordException;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.RegistryPathStatus;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.KerberosDiags;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.SliderApplicationApi;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.api.StateValues;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformationList;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.client.ipc.SliderApplicationIpcClient;
+import org.apache.slider.client.ipc.SliderClusterOperations;
+import org.apache.slider.common.Constants;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;

[13/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
new file mode 100644
index 000..37a8935
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
+import com.codahale.metrics.health.HealthCheckRegistry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Class for all metrics and monitoring
+ */
+public class MetricsAndMonitoring extends CompositeService {
+  protected static final Logger log =
+LoggerFactory.getLogger(MetricsAndMonitoring.class);
+  public MetricsAndMonitoring(String name) {
+super(name);
+  }
+  
+  public MetricsAndMonitoring() {
+super("MetricsAndMonitoring");
+  }
+  
+  /**
+   * Singleton of metrics registry
+   */
+  final MetricRegistry metrics = new MetricRegistry();
+
+  final HealthCheckRegistry health = new HealthCheckRegistry();
+
+  private final Map meterAndCounterMap
+  = new ConcurrentHashMap<>();
+
+  private final List metricSets = new ArrayList<>();
+
+  /**
+   * List of recorded events
+   */
+  private final List eventHistory = new ArrayList<>(100);
+
+  public static final int EVENT_LIMIT = 1000;
+
+  public MetricRegistry getMetrics() {
+return metrics;
+  }
+
+  public HealthCheckRegistry getHealth() {
+return health;
+  }
+
+  @Override
+  protected void serviceInit(Configuration conf) throws Exception {
+addService(new MetricsBindingService("MetricsBindingService",
+metrics));
+super.serviceInit(conf);
+  }
+
+  @Override
+  protected void serviceStop() throws Exception {
+super.serviceStop();
+for (MetricSet set : metricSets) {
+  unregister(set);
+}
+  }
+
+  public MeterAndCounter getMeterAndCounter(String name) {
+return meterAndCounterMap.get(name);
+  }
+
+  /**
+   * Get or create the meter/counter pair
+   * @param name name of instance
+   * @return an instance
+   */
+  public MeterAndCounter getOrCreateMeterAndCounter(String name) {
+MeterAndCounter instance = meterAndCounterMap.get(name);
+if (instance == null) {
+  synchronized (this) {
+// check in a sync block
+instance = meterAndCounterMap.get(name);
+if (instance == null) {
+  instance = new MeterAndCounter(metrics, name);
+  meterAndCounterMap.put(name, instance);
+}
+  }
+}
+return instance;
+  }
+
+  /**
+   * Get a specific meter and mark it. This will create and register it on 
demand.
+   * @param name name of meter/counter
+   */
+  public void markMeterAndCounter(String name) {
+MeterAndCounter meter = getOrCreateMeterAndCounter(name);
+meter.mark();
+  }
+
+  /**
+   * Given a {@link Metric}, registers it under the given name.
+   *
+   * @param name   the name of the metric
+   * @param metric the metric
+   * @param the type of the metric
+   * @return {@code metric}
+   * @throws IllegalArgumentException if the name is already registered
+   */
+  public  T register(String name, 

[11/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
new file mode 100644
index 000..16c2435
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -0,0 +1,2489 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricRegistry;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterDescriptionKeys;
+import org.apache.slider.api.ClusterDescriptionOperations;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderInternalStateException;
+import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.persist.AggregateConfSerDeser;
+import org.apache.slider.core.persist.ConfTreeSerDeser;
+import org.apache.slider.providers.PlacementPolicy;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.LongGauge;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.management.MetricsConstants;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import 

[06/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
new file mode 100644
index 000..80b7a5e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.agent;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RegistrationResponse {
+
+  @JsonProperty("response")
+  private RegistrationStatus response;
+
+  /**
+   * exitstatus is a code of error which was rised on server side. exitstatus
+   * = 0 (OK - Default) exitstatus = 1 (Registration failed because different
+   * version of agent and server)
+   */
+  @JsonProperty("exitstatus")
+  private int exitstatus;
+
+  /** log - message, which will be printed to agents log */
+  @JsonProperty("log")
+  private String log;
+
+  /** tags - tags associated with the container */
+  @JsonProperty("tags")
+  private String tags;
+  
+  @JsonProperty("package")
+  private String pkg;
+
+  //Response id to start with, usually zero.
+  @JsonProperty("responseId")
+  private long responseId;
+
+  @JsonProperty("statusCommands")
+  private List statusCommands = null;
+
+  public RegistrationResponse() {
+  }
+
+  public RegistrationStatus getResponse() {
+return response;
+  }
+
+  public void setResponse(RegistrationStatus response) {
+this.response = response;
+  }
+
+  public int getExitstatus() {
+return exitstatus;
+  }
+
+  public void setExitstatus(int exitstatus) {
+this.exitstatus = exitstatus;
+  }
+
+  public RegistrationStatus getResponseStatus() {
+return response;
+  }
+
+  public void setResponseStatus(RegistrationStatus response) {
+this.response = response;
+  }
+
+  public List getStatusCommands() {
+return statusCommands;
+  }
+
+  public void setStatusCommands(List statusCommands) {
+this.statusCommands = statusCommands;
+  }
+
+  public long getResponseId() {
+return responseId;
+  }
+
+  public void setResponseId(long responseId) {
+this.responseId = responseId;
+  }
+
+  public String getTags() {
+return tags;
+  }
+
+  public void setTags(String tags) {
+this.tags = tags;
+  }
+
+  public String getLog() {
+return log;
+  }
+
+  public void setLog(String log) {
+this.log = log;
+  }
+
+  public String getPkg() {
+return pkg;
+  }
+
+  public void setPkg(String pkg) {
+this.pkg = pkg;
+  }
+
+  @Override
+  public String toString() {
+return "RegistrationResponse{" +
+   "response=" + response +
+   ", responseId=" + responseId +
+   ", statusCommands=" + statusCommands +
+   '}';
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationStatus.java
 

[23/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
new file mode 100644
index 000..c3a1d0e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/RunService.java
@@ -0,0 +1,62 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.core.main;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service;
+
+/**
+ * An interface which services can implement to have their
+ * execution managed by the ServiceLauncher.
+ * The command line options will be passed down before the 
+ * {@link Service#init(Configuration)} operation is invoked via an
+ * invocation of {@link RunService#bindArgs(Configuration, String...)}
+ * After the service has been successfully started via {@link Service#start()}
+ * the {@link RunService#runService()} method is called to execute the 
+ * service. When this method returns, the service launcher will exit, using
+ * the return code from the method as its exit option.
+ */
+public interface RunService extends Service {
+
+  /**
+   * Propagate the command line arguments.
+   * This method is called before {@link Service#init(Configuration)};
+   * the configuration that is returned from this operation
+   * is the one that is passed on to the init operation.
+   * This permits implemenations to change the configuration before
+   * the init operation.n
+   * 
+   *
+   * @param config the initial configuration build up by the
+   * service launcher.
+   * @param args argument list list of arguments passed to the command line
+   * after any launcher-specific commands have been stripped.
+   * @return the configuration to init the service with. This MUST NOT be null.
+   * Recommended: pass down the config parameter with any changes
+   * @throws Exception any problem
+   */
+  Configuration bindArgs(Configuration config, String... args) throws 
Exception;
+  
+  /**
+   * Run a service. This called after {@link Service#start()}
+   * @return the exit code
+   * @throws Throwable any exception to report
+   */
+  int runService() throws Throwable ;
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
new file mode 100644
index 000..27813b7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/main/ServiceLaunchException.java
@@ -0,0 +1,73 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed 

[21/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
new file mode 100644
index 000..bf71861
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.restclient;
+
+import com.google.common.base.Preconditions;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+import org.apache.hadoop.conf.Configuration;
+import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
+import org.apache.slider.core.exceptions.ExceptionConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+/**
+ * Class to bond to a Jersey client, for UGI integration and SPNEGO.
+ * 
+ *   Usage: create an instance, then when creating a Jersey Client
+ *   pass in to the constructor the handler provided by {@link #getHandler()}
+ *
+ * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
+ */
+public class UgiJerseyBinding implements
+HttpURLConnectionFactory {
+  private static final Logger log =
+  LoggerFactory.getLogger(UgiJerseyBinding.class);
+
+  private final UrlConnectionOperations operations;
+  private final URLConnectionClientHandler handler;
+
+  /**
+   * Construct an instance
+   * @param operations operations instance
+   */
+  @SuppressWarnings("ThisEscapedInObjectConstruction")
+  public UgiJerseyBinding(UrlConnectionOperations operations) {
+Preconditions.checkArgument(operations != null, "Null operations");
+this.operations = operations;
+handler = new URLConnectionClientHandler(this);
+  }
+
+  /**
+   * Create an instance off the configuration. The SPNEGO policy
+   * is derived from the current UGI settings.
+   * @param conf config
+   */
+  public UgiJerseyBinding(Configuration conf) {
+this(new UrlConnectionOperations(conf));
+  }
+
+  /**
+   * Get a URL connection. 
+   * @param url URL to connect to
+   * @return the connection
+   * @throws IOException any problem. {@link AuthenticationException} 
+   * errors are wrapped
+   */
+  @Override
+  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+try {
+  // open a connection handling status codes and so redirections
+  // but as it opens a connection, it's less useful than you think.
+
+  return operations.openConnection(url);
+} catch (AuthenticationException e) {
+  throw new IOException(e);
+}
+  }
+
+  public UrlConnectionOperations getOperations() {
+return operations;
+  }
+
+  public URLConnectionClientHandler getHandler() {
+return handler;
+  }
+  
+  /**
+   * Get the SPNEGO flag (as found in the operations instance
+   * @return the spnego policy
+   */
+  public boolean isUseSpnego() {
+return operations.isUseSpnego();
+  }
+
+
+  /**
+   * Uprate error codes 400 and up into faults; 
+   * 
+   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
+   */
+  public static 

[04/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
new file mode 100644
index 000..9c14ca7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/HttpProbe.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.servicemonitor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+public class HttpProbe extends Probe {
+  protected static final Logger log = LoggerFactory.getLogger(HttpProbe.class);
+
+  private final URL url;
+  private final int timeout;
+  private final int min, max;
+
+
+  public HttpProbe(URL url, int timeout, int min, int max, Configuration conf) 
throws IOException {
+super("Http probe of " + url + " [" + min + "-" + max + "]", conf);
+this.url = url;
+this.timeout = timeout;
+this.min = min;
+this.max = max;
+  }
+
+  public static HttpURLConnection getConnection(URL url, int timeout) throws 
IOException {
+HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+connection.setInstanceFollowRedirects(true);
+connection.setConnectTimeout(timeout);
+return connection;
+  }
+  
+  @Override
+  public ProbeStatus ping(boolean livePing) {
+ProbeStatus status = new ProbeStatus();
+HttpURLConnection connection = null;
+try {
+  if (log.isDebugEnabled()) {
+// LOG.debug("Fetching " + url + " with timeout " + timeout);
+  }
+  connection = getConnection(url, this.timeout);
+  int rc = connection.getResponseCode();
+  if (rc < min || rc > max) {
+String error = "Probe " + url + " error code: " + rc;
+log.info(error);
+status.fail(this,
+new IOException(error));
+  } else {
+status.succeed(this);
+  }
+} catch (IOException e) {
+  String error = "Probe " + url + " failed: " + e;
+  log.info(error, e);
+  status.fail(this,
+  new IOException(error, e));
+} finally {
+  if (connection != null) {
+connection.disconnect();
+  }
+}
+return status;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
new file mode 100644
index 000..a1ad44f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/LogEntryBuilder.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in 

[01/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe) [Forced Update!]

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services f04eb0208 -> c22be358a (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
new file mode 100644
index 000..b8bdc59
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto
@@ -0,0 +1,396 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.slider.api.proto";
+option java_outer_classname = "Messages";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package org.apache.slider.api;
+
+//import "Security.proto";
+
+/*
+  Look at SliderClusterProtocol.proto to see how to build this
+*/
+
+message RoleInstanceState {
+  required string name =1;
+  optional string role =2;
+  required uint32 state =   4;
+  required uint32 exitCode =5;
+  optional string command = 6;
+  optional string diagnostics = 7;
+  repeated string output =  8;
+  repeated string environment = 9;
+  required uint32 roleId = 10;
+  required bool released = 11;
+  required int64 createTime =  12;
+  required int64 startTime =   13;
+  required string host =   14;
+  required string hostURL =15;
+  optional string appVersion = 16;
+}
+
+/**
+ * stop the cluster
+ */
+message StopClusterRequestProto {
+  /**
+  message to include
+  */
+  required string message = 1;
+}
+
+/**
+ * stop the cluster
+ */
+message StopClusterResponseProto {
+}
+
+/**
+ * upgrade the containers
+ */
+message UpgradeContainersRequestProto {
+  /**
+  message to include
+  */
+  required string message = 1;
+  repeated string container =   2;
+  repeated string component =   3;
+}
+
+/**
+ * upgrade the containers
+ */
+message UpgradeContainersResponseProto {
+}
+
+/**
+ * flex the cluster
+ */
+message FlexClusterRequestProto {
+  required string clusterSpec = 1;
+}
+
+
+/**
+ * flex the cluster
+ */
+message FlexClusterResponseProto {
+  required bool response = 1;
+}
+
+
+/**
+ * void request
+ */
+message GetJSONClusterStatusRequestProto {
+}
+
+/**
+ * response
+ */
+message GetJSONClusterStatusResponseProto {
+  required string clusterSpec = 1;
+}
+
+/**
+ * list the nodes in a role
+ */
+message ListNodeUUIDsByRoleRequestProto {
+  required string role = 1;
+}
+
+/**
+ * list the nodes in a role
+ */
+message ListNodeUUIDsByRoleResponseProto {
+  repeated string uuid = 1 ;
+}
+
+/**
+ * get a node
+ */
+message GetNodeRequestProto {
+  required string uuid = 1;
+}
+
+
+/**
+ * response on a node
+ */
+message GetNodeResponseProto {
+   required RoleInstanceState clusterNode = 1 ;
+}
+
+/**
+ * list the nodes for the UUDs
+ */
+message GetClusterNodesRequestProto {
+  repeated string uuid = 1 ;
+}
+
+/**
+ * list the nodes in a role
+ */
+message GetClusterNodesResponseProto {
+  repeated RoleInstanceState clusterNode = 1 ;
+}
+
+/**
+ * Echo
+ */
+message EchoRequestProto {
+  required string text = 1;
+}
+
+/**
+ * Echo reply
+ */
+message EchoResponseProto {
+  required string text = 1;
+}
+
+
+/**
+ * Kill a container
+ */
+message KillContainerRequestProto {
+  required string id = 1;
+}
+
+/**
+ * Kill reply
+ */
+message KillContainerResponseProto {
+  required bool success = 1;
+}
+
+/**
+ * AM suicide
+ */
+message AMSuicideRequestProto {
+  required string text =  1;
+  required int32 signal = 2;
+  required int32 delay =  3;
+}
+
+/**
+ * AM suicide reply. For this to be returned implies
+ * a failure of the AM to kill itself
+ */
+message AMSuicideResponseProto {
+
+}
+
+
+/**
+ * Ask for the instance definition details
+ */
+message 

[09/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
new file mode 100644
index 000..37e9a7f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+import org.apache.slider.server.services.utility.PatternValidator;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Implementation of {@link StateAccessForProviders}, which means
+ * state access for providers, web UI and IPC/REST views.
+ */
+public class ProviderAppState implements StateAccessForProviders {
+
+
+  private final Map publishedConfigSets =
+  new ConcurrentHashMap<>(5);
+  private final PublishedExportsSet publishedExportsSets = new 
PublishedExportsSet();
+  private static final PatternValidator validator = new PatternValidator(
+  RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP);
+  private String applicationName;
+
+  private final AppState appState;
+
+  public ProviderAppState(String applicationName, AppState appState) {
+this.appState = appState;
+this.applicationName = applicationName;
+  }
+
+  public void setApplicationName(String applicationName) {
+this.applicationName = applicationName;
+  }
+
+  @Override
+  public String getApplicationName() {
+return applicationName;
+  }
+
+  @Override
+  public PublishedConfigSet getPublishedSliderConfigurations() {
+return getOrCreatePublishedConfigSet(RestPaths.SLIDER_CONFIGSET);
+  }
+
+  @Override
+  public PublishedExportsSet getPublishedExportsSet() {
+return publishedExportsSets;
+  }
+
+  @Override
+  public PublishedConfigSet getPublishedConfigSet(String name) {
+return publishedConfigSets.get(name);
+  }
+
+  @Override
+  public PublishedConfigSet getOrCreatePublishedConfigSet(String name) {
+PublishedConfigSet set = publishedConfigSets.get(name);
+if (set == null) {
+  validator.validate(name);
+  synchronized (publishedConfigSets) {
+// synchronized double check to ensure that there is never an 
overridden
+// config set created
+set = publishedConfigSets.get(name);
+if (set == null) {
+  set = new PublishedConfigSet();
+  publishedConfigSets.put(name, set);
+}
+  }
+}
+return set;
+  }
+
+  @Override
+  public List listConfigSets() {
+
+synchronized (publishedConfigSets) {
+  List sets = new 

[08/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
new file mode 100644
index 000..ad91183
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.registry.docstore.PublishedConfigSet;
+import org.apache.slider.core.registry.docstore.PublishedExportsSet;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * The methods to offer state access to the providers and other parts of
+ * the system which want read-only access to the state.
+ */
+public interface StateAccessForProviders {
+
+  /**
+   * Get a map of role status entries by role Id
+   * @return the map of currently defined roles.
+   */
+  Map getRoleStatusMap();
+
+  /**
+   * Get the name of the application
+   * @return the name
+   */
+  String getApplicationName();
+
+  /**
+   * Get the published configurations
+   * @return the configuration set
+   */
+  PublishedConfigSet getPublishedSliderConfigurations();
+
+  /**
+   * Get the published exports set
+   * @return
+   */
+  PublishedExportsSet getPublishedExportsSet();
+
+  /**
+   * Get a named published config set
+   * @param name name to look up
+   * @return the instance or null
+   */
+  PublishedConfigSet getPublishedConfigSet(String name);
+
+  /**
+   * Get a named published config set, creating it if need be.
+   * @param name name to look up
+   * @return the instance -possibly a new one
+   */
+  PublishedConfigSet getOrCreatePublishedConfigSet(String name);
+
+  /**
+   * List the config sets -this takes a clone of the current set
+   * @return a list of config sets
+   */
+  List listConfigSets();
+
+  /**
+   * Get a map of all the failed containers
+   * @return map of recorded failed containers
+   */
+  Map getFailedContainers();
+
+  /**
+   * Get the live containers.
+   * 
+   * @return the live nodes
+   */
+  Map getLiveContainers();
+
+  /**
+   * Get the current cluster description 
+   * @return the actual state of the cluster
+   */
+  ClusterDescription getClusterStatus();
+
+  /**
+   * Get at the snapshot of the resource config
+   * Changes here do not affect the application state.
+   * @return the most recent settings
+   */
+  ConfTreeOperations getResourcesSnapshot();
+
+  /**
+   * Get at the snapshot of the appconf config
+   * Changes here do not affect the application state.
+   * @return the most recent settings
+   */
+  ConfTreeOperations getAppConfSnapshot();
+
+  /**
+   * Get at the snapshot of the internals config.
+   * Changes here do not 

[22/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
new file mode 100644
index 000..1bdfb9c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/YarnAppListClient.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.common.tools.CoreFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Client code for interacting with a list of service instances.
+ * The initial logic just enumerates service instances in the YARN RM
+ */
+public class YarnAppListClient {
+
+  private final SliderYarnClientImpl yarnClient;
+  private final String username;
+  private final Configuration conf;
+  private static final Logger log =
+  LoggerFactory.getLogger(YarnAppListClient.class);
+
+  public YarnAppListClient(SliderYarnClientImpl yarnClient,
+  String username,
+  Configuration conf) {
+
+Preconditions.checkArgument(yarnClient != null,
+"yarn client is null: is app inited?");
+Preconditions.checkArgument(username != null,
+"username is null");
+Preconditions.checkArgument(conf != null,
+"conf parameter is null");
+this.yarnClient = yarnClient;
+this.username = username;
+this.conf = conf;
+  }
+
+  /**
+   * find all live instances of a specific app -if there is more than one 
+   * in the cluster, this returns them all. State should be running or earlier
+   * in the lifecycle
+   * @param appname application name
+   * @return the list of all matching application instances
+   */
+  public List findAllLiveInstances(String appname)
+throws YarnException, IOException {
+return yarnClient.findAllLiveInstances(username, appname);
+  }
+
+
+  /**
+   * Find an instance of a application belong to the current user
+   * @param appname application name
+   * @return the app report or null if none is found
+   * @throws YarnException YARN issues
+   * @throws IOException IO problems
+   */
+  public ApplicationReport findInstance(String appname) throws
+YarnException,
+IOException {
+List instances = listInstances(null);
+return yarnClient.findClusterInInstanceList(instances, appname);
+  }
+
+  /**
+   * List instances belonging to the specific user
+   * @return a possibly empty list of AMs
+   */
+  public List listInstances()
+  throws YarnException, IOException {
+return listInstances(null);
+  }
+
+  /**
+   * List instances belonging to a specific user
+   * @return a possibly empty list of AMs
+   * @param user user if not the default. null means default, "" means all 
users, 
+   * otherwise it is the name of a user
+   */
+  public List listInstances(String user)
+  throws YarnException, IOException {
+String listUser = user == null ? username : user;
+

[20/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
new file mode 100644
index 000..9767430
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderCore.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.SliderException;
+
+import java.util.List;
+public interface ProviderCore {
+
+  String getName();
+
+  List getRoles();
+
+  Configuration getConf();
+
+  /**
+   * Verify that an instance definition is considered valid by the provider
+   * @param instanceDefinition instance definition
+   * @throws SliderException if the configuration is not valid
+   */
+  void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+  SliderException;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
new file mode 100644
index 000..761ac0f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers;
+
+import org.apache.slider.api.ResourceKeys;
+
+/**
+ * Provider role and key for use in app requests.
+ * 
+ * This class uses the role name as the key for hashes and in equality tests,
+ * and ignores the other values.
+ */
+public final class ProviderRole {
+  public final String name;
+  public final String group;
+  public final int id;
+  public int placementPolicy;
+  public int nodeFailureThreshold;
+  public final long placementTimeoutSeconds;
+  public final String labelExpression;
+
+  public ProviderRole(String name, int id) {
+this(name,
+name,
+id,
+PlacementPolicy.DEFAULT,
+ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
+ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
+ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
+  }
+
+  /**
+   * Create a provider role
+   * @param name role/component name
+   

[19/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
new file mode 100644
index 000..4ffae7c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -0,0 +1,3212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+import org.apache.hadoop.registry.client.types.Endpoint;
+import org.apache.hadoop.registry.client.types.ProtocolTypes;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.NoSuchNodeException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.ConfigUtils;
+import org.apache.slider.core.registry.docstore.ExportEntry;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
+import org.apache.slider.core.registry.docstore.PublishedExports;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.MonitorDetail;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import 
org.apache.slider.providers.agent.application.metadata.AbstractComponent;
+import org.apache.slider.providers.agent.application.metadata.Application;
+import 

[10/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
new file mode 100644
index 000..a8aa1a2
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Binding information for application states; designed to be extensible
+ * so that tests don't have to be massivley reworked when new arguments
+ * are added.
+ */
+public class AppStateBindingInfo {
+  public AggregateConf instanceDefinition;
+  public Configuration serviceConfig = new Configuration();
+  public Configuration publishedProviderConf = new Configuration(false);
+  public List roles = new ArrayList<>();
+  public FileSystem fs;
+  public Path historyPath;
+  public List liveContainers = new ArrayList<>(0);
+  public Map applicationInfo = new HashMap<>();
+  public ContainerReleaseSelector releaseSelector = new 
SimpleReleaseSelector();
+  /** node reports off the RM. */
+  public List nodeReports = new ArrayList<>(0);
+
+  public void validate() throws IllegalArgumentException {
+Preconditions.checkArgument(instanceDefinition != null, "null 
instanceDefinition");
+Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig");
+Preconditions.checkArgument(publishedProviderConf != null, "null 
publishedProviderConf");
+Preconditions.checkArgument(releaseSelector != null, "null 
releaseSelector");
+Preconditions.checkArgument(roles != null, "null providerRoles");
+Preconditions.checkArgument(fs != null, "null fs");
+Preconditions.checkArgument(historyPath != null, "null historyDir");
+Preconditions.checkArgument(nodeReports != null, "null nodeReports");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
new file mode 100644
index 000..5b3a93c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationOutcome.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF 

[14/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
new file mode 100644
index 000..6b61681
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionFlexCluster.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+public class ActionFlexCluster extends AsyncAction {
+
+  public final ConfTree resources;
+  
+  public ActionFlexCluster(String name,
+  long delay,
+  TimeUnit timeUnit, ConfTree resources) {
+super(name, delay, timeUnit, ATTR_CHANGES_APP_SIZE);
+this.resources = resources;
+  }
+
+  @Override
+  public void execute(SliderAppMaster appMaster,
+  QueueAccess queueService,
+  AppState appState) throws Exception {
+appMaster.flexCluster(resources);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
new file mode 100644
index 000..e2ad559
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.actions;
+
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.slider.server.appmaster.SliderAppMaster;
+import org.apache.slider.server.appmaster.state.AppState;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Exit an emergency JVM halt.
+ * @see ExitUtil#halt(int, String) 
+ */
+public class ActionHalt extends AsyncAction {
+
+  private final int status;
+  private final String text;
+
+  public ActionHalt(
+  int status,
+  String text,
+  long delay, TimeUnit timeUnit) {
+
+// do not declare that this action halts the cluster ... keep it a surprise
+super("Halt", delay, timeUnit);
+this.status = status;
+this.text = 

[16/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
new file mode 100644
index 000..67d3647
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.slideram;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.ConfigHelper;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.PublishedConfiguration;
+import org.apache.slider.core.registry.info.CustomRegistryConstants;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.server.appmaster.PublishedArtifacts;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
+
+/**
+ * Exists just to move some functionality out of AppMaster into a peer class
+ * of the actual service provider doing the real work
+ */
+public class SliderAMProviderService extends AbstractProviderService implements
+ProviderCore,
+AgentKeys,
+SliderKeys {
+
+  public SliderAMProviderService() {
+super("SliderAMProviderService");
+  }
+
+  @Override
+  public String getHumanName() {
+return "Slider Application";
+  }
+  
+  @Override
+  public Configuration loadProviderConfigurationInformation(File confDir) 
throws
+  BadCommandArgumentsException,
+  IOException {
+return null;
+  }
+
+  @Override
+  public void buildContainerLaunchContext(ContainerLauncher containerLauncher,
+  AggregateConf instanceDefinition,
+  Container container,
+  ProviderRole role,
+  SliderFileSystem sliderFileSystem,
+  Path generatedConfPath,
+  MapOperations resourceComponent,
+  MapOperations appComponent,
+  Path containerTmpDirPath) throws IOException, SliderException {
+  }
+
+  @Override
+  public List getRoles() {
+return new ArrayList<>(0);
+  }
+
+  @Override
+  public void validateInstanceDefinition(AggregateConf instanceDefinition) 
throws
+  SliderException {
+
+  }
+
+  @Override
+  public void applyInitialRegistryDefinitions(URL amWebURI,
+  URL agentOpsURI,
+  URL agentStatusURI,
+  ServiceRecord serviceRecord)
+  throws IOException {
+super.applyInitialRegistryDefinitions(amWebURI,
+agentOpsURI,
+agentStatusURI,
+

[12/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
new file mode 100644
index 000..dd4785d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/RpcBinder.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.rpc;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolProxy;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.RpcEngine;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.slider.api.SliderClusterProtocol;
+import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.slider.core.exceptions.ServiceNotReadyException;
+import org.apache.slider.core.exceptions.SliderException;
+
+import static org.apache.slider.common.SliderXmlConfKeys.*; 
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+public class RpcBinder {
+  protected static final Logger log =
+LoggerFactory.getLogger(RpcBinder.class);
+
+  /**
+   * Create a protobuf server bonded to the specific socket address
+   * @param addr address to listen to; 0.0.0.0 as hostname acceptable
+   * @param conf config
+   * @param secretManager token secret handler
+   * @param numHandlers threads to service requests
+   * @param blockingService service to handle
+   * @param portRangeConfig range of ports
+   * @return the IPC server itself
+   * @throws IOException
+   */
+  public static Server createProtobufServer(InetSocketAddress addr,
+Configuration conf,
+SecretManager secretManager,
+int numHandlers,
+BlockingService blockingService,
+String portRangeConfig) throws
+  IOException {
+Class sliderClusterAPIClass = registerSliderAPI(
+conf);
+RPC.Server server = new 
RPC.Builder(conf).setProtocol(sliderClusterAPIClass)
+ .setInstance(blockingService)
+   

[05/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
new file mode 100644
index 000..a8e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ComponentResource.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class ComponentResource {
+  private final Map props;
+  private String href;
+
+  public ComponentResource() {
+this(null, null, null, null);
+  }
+
+  public ComponentResource(String name,
+   Map props,
+   UriBuilder uriBuilder,
+   Map pathElems) {
+this.props = props;
+  }
+
+  public Map getProps() {
+return props;
+  }
+
+  public String getHref() {
+return href;
+  }
+
+  public void setHref(String href) {
+this.href = href;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
new file mode 100644
index 000..407bab6
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/ConfTreeResource.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.appmaster.web.rest.management.resources;
+
+import org.apache.slider.core.conf.ConfTree;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import javax.ws.rs.core.UriBuilder;
+import java.util.Map;
+

[02/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
new file mode 100644
index 000..9e9e7ac
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -0,0 +1,598 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.server.services.workflow;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.io.IOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Execute a long-lived process.
+ *
+ * 
+ * Hadoop's {@link org.apache.hadoop.util.Shell} class assumes it is executing
+ * a short lived application; this class allows for the process to run for the
+ * life of the Java process that forked it.
+ * It is designed to be embedded inside a YARN service, though this is not
+ * the sole way that it can be used
+ * 
+ * Key Features:
+ * 
+ *   Output is streamed to the output logger provided.
+ *   the input stream is closed as soon as the process starts.
+ *   The most recent lines of output are saved to a linked list.
+ *   A synchronous callback, {@link LongLivedProcessLifecycleEvent},
+ *   is raised on the start and finish of a process.
+ * 
+ * 
+ */
+public class LongLivedProcess implements Runnable {
+  /**
+   * Limit on number of lines to retain in the "recent" line list:{@value}
+   */
+  public static final int RECENT_LINE_LOG_LIMIT = 64;
+
+  /**
+   * Const defining the time in millis between polling for new text.
+   */
+  private static final int STREAM_READER_SLEEP_TIME = 200;
+  
+  /**
+   * limit on the length of a stream before it triggers an automatic newline.
+   */
+  private static final int LINE_LENGTH = 256;
+  private final ProcessBuilder processBuilder;
+  private Process process;
+  private Integer exitCode = null;
+  private final String name;
+  private final ExecutorService processExecutor;
+  private final ExecutorService logExecutor;
+  
+  private ProcessStreamReader processStreamReader;
+  //list of recent lines, recorded for extraction into reports
+  private final List recentLines = new LinkedList<>();
+  private int recentLineLimit = RECENT_LINE_LOG_LIMIT;
+  private LongLivedProcessLifecycleEvent lifecycleCallback;
+  private final AtomicBoolean finalOutputProcessed = new AtomicBoolean(false);
+
+  /**
+   * Log supplied in the constructor for the spawned process -accessible
+   * to inner classes
+   */
+  private Logger processLog;
+  
+  /**
+   * Class log -accessible to inner classes
+   */
+  private static final Logger LOG = 
LoggerFactory.getLogger(LongLivedProcess.class);
+
+  /**
+   *  flag to indicate that the process is done
+   */
+  private final AtomicBoolean finished = new AtomicBoolean(false);
+
+  /**
+   * Create an instance
+   * @param name process name
+   * @param processLog log for output (or null)
+   * @param commands command list
+   */
+  public LongLivedProcess(String name,
+  Logger processLog,
+  List commands) {
+Preconditions.checkArgument(commands != 

[03/74] [abbrv] hadoop git commit: YARN-5461. Initial code ported from slider-core module. (jianhe)

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/31c4a419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
new file mode 100644
index 000..e436ae9
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java
@@ -0,0 +1,495 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.server.services.security;
+
+import com.google.inject.Singleton;
+import org.apache.commons.io.FileUtils;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.charset.Charset;
+import java.text.MessageFormat;
+
+@Singleton
+public class CertificateManager {
+
+  private static final Logger LOG =
+  LoggerFactory.getLogger(CertificateManager.class);
+
+  private static final String GEN_SRVR_KEY = "openssl genrsa -des3 " +
+  "-passout pass:{0} -out {1}" + File.separator + "{2} 4096 ";
+  private static final String GEN_SRVR_REQ = "openssl req -passin pass:{0} " +
+  "-new -key {1}" + File.separator + "{2} -out {1}" + File.separator +
+  "{5} -config {1}" + File.separator + "ca.config " +
+  "-subj {6} -batch";
+  private static final String SIGN_SRVR_CRT = "openssl ca -create_serial " +
+"-out {1}" + File.separator + "{3} -days 365 -keyfile {1}" + File.separator
++ "{2} -key {0} -selfsign -extensions jdk7_ca -config {1}" + File.separator
++ "ca.config -batch -infiles {1}" + File.separator + "{5}";
+  private static final String EXPRT_KSTR = "openssl pkcs12 -export" +
+  " -in {2}" + File.separator + "{4} -inkey {2}" + File.separator +
+  "{3} -certfile {2}" + File.separator + "{4} -out {2}" + File.separator +
+  "{5} -password pass:{1} -passin pass:{0} \n";
+  private static final String REVOKE_AGENT_CRT = "openssl ca " +
+  "-config {0}" + File.separator + "ca.config -keyfile {0}" +
+  File.separator + "{4} -revoke {0}" + File.separator + "{2} -batch " +
+  "-passin pass:{3} -cert {0}" + File.separator + "{5}";
+  private static final String SIGN_AGENT_CRT = "openssl ca -config " +
+  "{0}" + File.separator + "ca.config -in {0}" + File.separator +
+  "{1} -out {0}" + File.separator + "{2} -batch -passin pass:{3} " +
+  "-keyfile {0}" + File.separator + "{4} -cert {0}" + File.separator + 
"{5}";
+  private static final String GEN_AGENT_KEY="openssl req -new -newkey " +
+  "rsa:1024 -nodes -keyout {0}" + File.separator +
+  "{2}.key -subj {1} -out {0}" + File.separator + "{2}.csr " +
+  "-config {3}" + File.separator + "ca.config ";
+  private String passphrase;
+  private String applicationName;
+
+
+  public void initialize(MapOperations compOperations) throws SliderException {
+String hostname = null;
+try {
+  hostname = InetAddress.getLocalHost().getCanonicalHostName();
+} catch (UnknownHostException e) {
+  hostname = "localhost";
+}
+this.initialize(compOperations, hostname, null, null);
+  }
+
+  /**
+* Verify that root certificate exists, generate it otherwise.
+*/
+  public void initialize(MapOperations compOperations,
+ String hostname, String containerId,
+ String appName) 

hadoop git commit: HADOOP-13886. s3guard: ITestS3AFileOperationCost.testFakeDirectoryDeletion failure. Contributed by Mingliang Liu

[liuml07]

Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-13345 49965d2d6 -> b2c4f8cfe


HADOOP-13886. s3guard: ITestS3AFileOperationCost.testFakeDirectoryDeletion 
failure. Contributed by Mingliang Liu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2c4f8cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2c4f8cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2c4f8cf

Branch: refs/heads/HADOOP-13345
Commit: b2c4f8cfe181d050e54ac29c08d5c0ac888dd4d0
Parents: 49965d2
Author: Mingliang Liu 
Authored: Mon Dec 12 11:58:11 2016 -0800
Committer: Mingliang Liu 
Committed: Tue Dec 13 13:45:32 2016 -0800

--
 .../org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2c4f8cf/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
index 3e2d2b0..98184e4 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -33,8 +33,8 @@ import java.net.URI;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
 import static org.apache.hadoop.fs.s3a.Statistic.*;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
-import static org.apache.hadoop.fs.s3a.S3ATestUtils.MetricDiff;
 import static org.apache.hadoop.test.GenericTestUtils.getTestDir;
+import static org.junit.Assume.assumeFalse;
 
 /**
  * Use metrics to assert about the cost of file status queries.
@@ -195,6 +195,13 @@ public class ITestS3AFileOperationCost extends 
AbstractS3ATestBase {
 + "In S3, rename deletes any fake directories as a part of "
 + "clean up activity");
 S3AFileSystem fs = getFileSystem();
+
+// As this test uses the s3 metrics to count the number of fake directory
+// operations, it depends on side effects happening internally. With
+// metadata store enabled, it is brittle to change. We disable this test
+// before the internal behavior w/ or w/o metadata store.
+assumeFalse(fs.isMetadataStoreConfigured());
+
 Path srcBaseDir = path("src");
 mkdirs(srcBaseDir);
 MetricDiff deleteRequests =


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-4438. Implement RM leader election with curator. Contributed by Jian He

[junping_du]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 6a18ae849 -> 120f3a0ff


YARN-4438. Implement RM leader election with curator. Contributed by Jian He

(cherry picked from commit 89022f8d4bac0e9d0b848fd91e9c4d700fe1cdbe)
(cherry picked from commit 2cbbf76c3dd9570e08914077de8e610d93ee0c9e)

Conflicts:

hadoop-yarn-project/CHANGES.txt


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/120f3a0f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/120f3a0f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/120f3a0f

Branch: refs/heads/branch-2.8
Commit: 120f3a0ff9da6252aa8004ee43242d75d36805f0
Parents: 6a18ae8
Author: Xuan 
Authored: Thu Jan 7 14:33:06 2016 -0800
Committer: Junping Du 
Committed: Tue Dec 13 13:30:15 2016 -0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |   5 +
 .../yarn/conf/TestYarnConfigurationFields.java  |   1 +
 .../server/resourcemanager/AdminService.java|  49 +++-
 .../resourcemanager/LeaderElectorService.java   | 144 ++
 .../yarn/server/resourcemanager/RMContext.java  |   4 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  24 +-
 .../TestLeaderElectorService.java   | 269 +++
 .../yarn/server/resourcemanager/TestRMHA.java   |   8 +-
 9 files changed, 494 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/120f3a0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5ce171c..8f2d02c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -550,6 +550,11 @@ public class YarnConfiguration extends Configuration {
   public static final String RM_HA_FC_ELECTOR_ZK_RETRIES_KEY = RM_HA_PREFIX
   + "failover-controller.active-standby-elector.zk.retries";
 
+  @Private
+  public static final String CURATOR_LEADER_ELECTOR =
+  RM_HA_PREFIX + "curator-leader-elector.enabled";
+  public static final boolean DEFAULT_CURATOR_LEADER_ELECTOR_ENABLED = false;
+
   
   // RM state store configs
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/120f3a0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index dcfbd5d..65e1c9c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -90,6 +90,7 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 .add(YarnConfiguration.DEFAULT_SHARED_CACHE_CHECKSUM_ALGO_IMPL);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE);
+
configurationPropsToSkipCompare.add(YarnConfiguration.CURATOR_LEADER_ELECTOR);
 
 // Ignore blacklisting nodes for AM failures feature since it is still a
 // "work in progress"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/120f3a0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 20ce76b..82e8578 100644
--- 

[4/6] hadoop git commit: HDFS-11226. cacheadmin, cryptoadmin and storagepolicyadmin should support generic options. Contributed by Brahma Reddy Battula

[xgong]

HDFS-11226. cacheadmin, cryptoadmin and storagepolicyadmin should support 
generic options. Contributed by Brahma Reddy Battula


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/754f15ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/754f15ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/754f15ba

Branch: refs/heads/YARN-5734
Commit: 754f15bae61b81ad3c2e3f722d1feaebf374e2c4
Parents: 2d4731c
Author: Mingliang Liu 
Authored: Mon Dec 12 17:36:52 2016 -0800
Committer: Mingliang Liu 
Committed: Mon Dec 12 18:21:15 2016 -0800

--
 .../main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java   | 8 ++--
 .../main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java  | 8 ++--
 .../org/apache/hadoop/hdfs/tools/StoragePolicyAdmin.java | 6 +-
 3 files changed, 17 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/754f15ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index 270a662..522f701 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
 
 import com.google.common.base.Joiner;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This class implements command-line operations on the HDFS Cache.
@@ -64,6 +65,7 @@ public class CacheAdmin extends Configured implements Tool {
   public int run(String[] args) throws IOException {
 if (args.length == 0) {
   AdminHelper.printUsage(false, "cacheadmin", COMMANDS);
+  ToolRunner.printGenericCommandUsage(System.err);
   return 1;
 }
 AdminHelper.Command command = AdminHelper.determineCommand(args[0],
@@ -74,6 +76,7 @@ public class CacheAdmin extends Configured implements Tool {
 System.err.println("Command names must start with dashes.");
   }
   AdminHelper.printUsage(false, "cacheadmin", COMMANDS);
+  ToolRunner.printGenericCommandUsage(System.err);
   return 1;
 }
 List argsList = new LinkedList();
@@ -88,9 +91,10 @@ public class CacheAdmin extends Configured implements Tool {
 }
   }
 
-  public static void main(String[] argsArray) throws IOException {
+  public static void main(String[] argsArray) throws Exception {
 CacheAdmin cacheAdmin = new CacheAdmin(new Configuration());
-System.exit(cacheAdmin.run(argsArray));
+int res = ToolRunner.run(cacheAdmin, argsArray);
+System.exit(res);
   }
 
   private static CacheDirectiveInfo.Expiration parseExpirationString(String 
ttlString)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/754f15ba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index b78da31..225f11a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.protocol.EncryptionZone;
 import org.apache.hadoop.tools.TableListing;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
 
 /**
  * This class implements crypto command-line operations.
@@ -54,6 +55,7 @@ public class CryptoAdmin extends Configured implements Tool {
   public int run(String[] args) throws IOException {
 if (args.length == 0) {
   AdminHelper.printUsage(false, "crypto", COMMANDS);
+  ToolRunner.printGenericCommandUsage(System.err);
   return 1;
 }
 final AdminHelper.Command command = AdminHelper.determineCommand(args[0],
@@ -64,6 +66,7 @@ public class CryptoAdmin extends Configured implements Tool {
 System.err.println("Command names must start with dashes.");
   }
   AdminHelper.printUsage(false, "crypto", COMMANDS);
+  ToolRunner.printGenericCommandUsage(System.err);
   return 1;
 }
 final List argsList = new LinkedList();
@@ -78,9 +81,10 @@ public class CryptoAdmin extends Configured implements Tool {
 }
   }
 
-  

[2/6] hadoop git commit: HADOOP-13871. ITestS3AInputStreamPerformance.testTimeToOpenAndReadWholeFileBlocks performance awful. Contributed by Steve Loughran

[xgong]

HADOOP-13871. 
ITestS3AInputStreamPerformance.testTimeToOpenAndReadWholeFileBlocks performance 
awful. Contributed by Steve Loughran


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6a39232
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6a39232
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6a39232

Branch: refs/heads/YARN-5734
Commit: c6a39232456fa0c98b2b9b6dbeaec762294ca01e
Parents: f66f618
Author: Mingliang Liu 
Authored: Mon Dec 12 14:55:34 2016 -0800
Committer: Mingliang Liu 
Committed: Mon Dec 12 14:55:34 2016 -0800

--
 .../apache/hadoop/fs/s3a/S3AInputStream.java| 33 ++---
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  3 ++
 .../tools/hadoop-aws/troubleshooting_s3a.md | 52 
 .../scale/ITestS3AInputStreamPerformance.java   | 47 --
 .../hadoop/fs/s3a/scale/S3AScaleTestBase.java   | 20 +---
 5 files changed, 140 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6a39232/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index dd6cdd7..3c4093d 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -132,7 +132,7 @@ public class S3AInputStream extends FSInputStream 
implements CanSetReadahead {
   throws IOException {
 
 if (wrappedStream != null) {
-  closeStream("reopen(" + reason + ")", contentRangeFinish);
+  closeStream("reopen(" + reason + ")", contentRangeFinish, false);
 }
 
 contentRangeFinish = calculateRequestLimit(inputPolicy, targetPos,
@@ -257,7 +257,7 @@ public class S3AInputStream extends FSInputStream 
implements CanSetReadahead {
 
 // if the code reaches here, the stream needs to be reopened.
 // close the stream; if read the object will be opened at the new pos
-closeStream("seekInStream()", this.contentRangeFinish);
+closeStream("seekInStream()", this.contentRangeFinish, false);
 pos = targetPos;
   }
 
@@ -414,7 +414,7 @@ public class S3AInputStream extends FSInputStream 
implements CanSetReadahead {
   closed = true;
   try {
 // close or abort the stream
-closeStream("close() operation", this.contentRangeFinish);
+closeStream("close() operation", this.contentRangeFinish, false);
 // this is actually a no-op
 super.close();
   } finally {
@@ -431,17 +431,17 @@ public class S3AInputStream extends FSInputStream 
implements CanSetReadahead {
* an abort.
*
* This does not set the {@link #closed} flag.
-   *
* @param reason reason for stream being closed; used in messages
* @param length length of the stream.
+   * @param forceAbort force an abort; used if explicitly requested.
*/
-  private void closeStream(String reason, long length) {
+  private void closeStream(String reason, long length, boolean forceAbort) {
 if (wrappedStream != null) {
 
   // if the amount of data remaining in the current request is greater
   // than the readahead value: abort.
   long remaining = remainingInCurrentRequest();
-  boolean shouldAbort = remaining > readahead;
+  boolean shouldAbort = forceAbort || remaining > readahead;
   if (!shouldAbort) {
 try {
   // clean close. This will read to the end of the stream,
@@ -470,6 +470,27 @@ public class S3AInputStream extends FSInputStream 
implements CanSetReadahead {
 }
   }
 
+  /**
+   * Forcibly reset the stream, by aborting the connection. The next
+   * {@code read()} operation will trigger the opening of a new HTTPS
+   * connection.
+   *
+   * This is potentially very inefficient, and should only be invoked
+   * in extreme circumstances. It logs at info for this reason.
+   * @return true if the connection was actually reset.
+   * @throws IOException if invoked on a closed stream.
+   */
+  @InterfaceStability.Unstable
+  public synchronized boolean resetConnection() throws IOException {
+checkNotClosed();
+boolean connectionOpen = wrappedStream != null;
+if (connectionOpen) {
+  LOG.info("Forced reset of connection to {}", uri);
+  closeStream("reset()", contentRangeFinish, true);
+}
+return connectionOpen;
+  }
+
   @Override
   public synchronized int available() throws IOException {
 checkNotClosed();

[6/6] hadoop git commit: HADOOP-13900. Remove snapshot version of SDK dependency from Azure Data Lake Store File System. Contributed by Vishwajeet Dusane

[xgong]

HADOOP-13900. Remove snapshot version of SDK dependency from Azure Data Lake 
Store File System. Contributed by Vishwajeet Dusane


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef34bf2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef34bf2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef34bf2b

Branch: refs/heads/YARN-5734
Commit: ef34bf2bb92a4e8def6617b185ae72db81450de8
Parents: b0b033e
Author: Mingliang Liu 
Authored: Tue Dec 13 10:34:32 2016 -0800
Committer: Mingliang Liu 
Committed: Tue Dec 13 10:34:32 2016 -0800

--
 hadoop-tools/hadoop-azure-datalake/pom.xml | 10 +-
 1 file changed, 1 insertion(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef34bf2b/hadoop-tools/hadoop-azure-datalake/pom.xml
--
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml 
b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 7f872c3..b2d419f 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -34,14 +34,6 @@
 UTF-8
 true
   
-  
-
-  snapshots-repo
-  https://oss.sonatype.org/content/repositories/snapshots
-  false
-  true
-
-  
   
 
   
@@ -128,7 +120,7 @@
 
   com.microsoft.azure
   azure-data-lake-store-sdk
-  2.0.4-SNAPSHOT
+  2.0.11
 
 
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[5/6] hadoop git commit: MAPREDUCE-6821. Fix javac warning related to the deprecated APIs after upgrading Jackson. Contributed by Yiqin Lin.

[xgong]

MAPREDUCE-6821. Fix javac warning related to the deprecated APIs after 
upgrading Jackson. Contributed by Yiqin Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0b033ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0b033ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0b033ea

Branch: refs/heads/YARN-5734
Commit: b0b033ea2e462356b8bbcf7790953ac09c712430
Parents: 754f15b
Author: Akira Ajisaka 
Authored: Tue Dec 13 14:20:02 2016 +0900
Committer: Akira Ajisaka 
Committed: Tue Dec 13 14:22:07 2016 +0900

--
 .../src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java| 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0b033ea/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index a842649..09edd94 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -72,7 +72,7 @@ import com.google.common.base.Charsets;
 class JobSubmitter {
   protected static final Log LOG = LogFactory.getLog(JobSubmitter.class);
   private static final ObjectReader READER =
-  new ObjectMapper().reader(Map.class);
+  new ObjectMapper().readerFor(Map.class);
   private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
   private static final int SHUFFLE_KEY_LENGTH = 64;
   private FileSystem jtFs;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[1/6] hadoop git commit: HADOOP-13852 hadoop build to allow hadoop version property to be explicitly set. Contriibuted by Steve Loughran

[xgong]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-5734 4c38f11ce -> ef34bf2bb


HADOOP-13852 hadoop build to allow hadoop version property to be explicitly 
set. Contriibuted by Steve Loughran


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f66f6189
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f66f6189
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f66f6189

Branch: refs/heads/YARN-5734
Commit: f66f61892ac7cbd57e8323d0a2a90cf58e9286ce
Parents: 4c38f11
Author: Steve Loughran 
Authored: Mon Dec 12 13:58:27 2016 +
Committer: Steve Loughran 
Committed: Mon Dec 12 13:58:27 2016 +

--
 BUILDING.txt | 11 ++-
 .../src/main/resources/common-version-info.properties|  4 ++--
 hadoop-project/pom.xml   |  3 +++
 .../src/main/resources/yarn-version-info.properties  |  2 +-
 4 files changed, 16 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f66f6189/BUILDING.txt
--
diff --git a/BUILDING.txt b/BUILDING.txt
index 8b2bba6..7afc3f0 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -390,7 +390,7 @@ http://www.zlib.net/
 
--
 Building distributions:
 
- * Build distribution with native code: mvn package 
[-Pdist][-Pdocs][-Psrc][-Dtar]
+ * Build distribution with native code: mvn package 
[-Pdist][-Pdocs][-Psrc][-Dtar][-Dmaven.javadoc.skip=true]
 
 
--
 Running compatibility checks with checkcompatibility.py
@@ -402,3 +402,12 @@ managers to compare the compatibility of a previous and 
current release.
 As an example, this invocation will check the compatibility of interfaces 
annotated as Public or LimitedPrivate:
 
 ./dev-support/bin/checkcompatibility.py --annotation 
org.apache.hadoop.classification.InterfaceAudience.Public --annotation 
org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate --include 
"hadoop.*" branch-2.7.2 trunk
+
+--
+Changing the Hadoop version declared returned by VersionInfo
+
+If for compatibility reasons the version of Hadoop has to be declared as a 2.x 
release in the information returned by
+org.apache.hadoop.util.VersionInfo, set the property declared.hadoop.version 
to the desired version.
+For example: mvn package -Pdist -Ddeclared.hadoop.version=2.11
+
+If unset, the project version declared in the POM file is used.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f66f6189/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
 
b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
index ad9a24d..9b74960 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
+++ 
b/hadoop-common-project/hadoop-common/src/main/resources/common-version-info.properties
@@ -16,11 +16,11 @@
 # limitations under the License.
 #
 
-version=${pom.version}
+version=${declared.hadoop.version}
 revision=${version-info.scm.commit}
 branch=${version-info.scm.branch}
 user=${user.name}
 date=${version-info.build.time}
 url=${version-info.scm.uri}
 srcChecksum=${version-info.source.md5}
-protocVersion=${protobuf.version}
\ No newline at end of file
+protocVersion=${protobuf.version}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f66f6189/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 336231e..9ec24ea 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -126,6 +126,9 @@
 1.0-alpha-8
 900
 1.11.45
+
+${pom.version}
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f66f6189/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
index 9a8575c..ee6f13d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-version-info.properties
+++ 

[3/6] hadoop git commit: HDFS-11233. Fix javac warnings related to the deprecated APIs after upgrading Jackson. Contributed by Yiqun Lin.

[xgong]

HDFS-11233. Fix javac warnings related to the deprecated APIs after upgrading 
Jackson. Contributed by Yiqun Lin.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d4731c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d4731c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d4731c0

Branch: refs/heads/YARN-5734
Commit: 2d4731c067ff64cd88f496eac8faaf302faa2ccc
Parents: c6a3923
Author: Akira Ajisaka 
Authored: Tue Dec 13 11:11:15 2016 +0900
Committer: Akira Ajisaka 
Committed: Tue Dec 13 11:11:15 2016 +0900

--
 .../hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java| 2 +-
 .../hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java  | 4 ++--
 .../org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java | 2 +-
 .../main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java | 2 +-
 .../java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java   | 2 +-
 .../web/oauth2/ConfRefreshTokenBasedAccessTokenProvider.java | 2 +-
 .../hdfs/web/oauth2/CredentialBasedAccessTokenProvider.java  | 2 +-
 .../hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java| 2 +-
 .../hadoop/hdfs/server/diskbalancer/command/Command.java | 2 +-
 .../server/diskbalancer/connectors/JsonNodeConnector.java| 2 +-
 .../server/diskbalancer/datamodel/DiskBalancerCluster.java   | 2 +-
 .../server/diskbalancer/datamodel/DiskBalancerVolume.java| 2 +-
 .../hadoop/hdfs/server/diskbalancer/planner/NodePlan.java| 2 +-
 .../test/java/org/apache/hadoop/hdfs/web/TestJsonUtil.java   | 8 
 14 files changed, 18 insertions(+), 18 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4731c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
index 0cdd107..505273e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
@@ -37,7 +37,7 @@ import java.io.IOException;
 public class DiskBalancerWorkItem {
   private static final ObjectMapper MAPPER = new ObjectMapper();
   private static final ObjectReader READER =
-  new ObjectMapper().reader(DiskBalancerWorkItem.class);
+  new ObjectMapper().readerFor(DiskBalancerWorkItem.class);
 
   private  long startTime;
   private long secondsElapsed;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4731c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
index 452ab27..af34c49 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkStatus.java
@@ -43,9 +43,9 @@ public class DiskBalancerWorkStatus {
   private static final ObjectMapper MAPPER_WITH_INDENT_OUTPUT =
   new ObjectMapper().enable(SerializationFeature.INDENT_OUTPUT);
   private static final ObjectReader READER_WORKSTATUS =
-  new ObjectMapper().reader(DiskBalancerWorkStatus.class);
+  new ObjectMapper().readerFor(DiskBalancerWorkStatus.class);
   private static final ObjectReader READER_WORKENTRY = new ObjectMapper()
-  .reader(defaultInstance().constructCollectionType(List.class,
+  .readerFor(defaultInstance().constructCollectionType(List.class,
   DiskBalancerWorkEntry.class));
 
   private final List currentState;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d4731c0/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/CombinedHostsFileReader.java
index 668d56e..8da5655 100644
--- 

hadoop git commit: HADOOP-13898. Should set HADOOP_JOB_HISTORYSERVER_HEAPSIZE only if it's empty. Contributed by Fei Hui.

[raviprak]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 282a56243 -> d7d9006af


HADOOP-13898. Should set HADOOP_JOB_HISTORYSERVER_HEAPSIZE only if it's empty. 
Contributed by Fei Hui.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7d9006a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7d9006a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7d9006a

Branch: refs/heads/branch-2
Commit: d7d9006afc60dd17884d939921a67d4d19f87f4f
Parents: 282a562
Author: Ravi Prakash 
Authored: Tue Dec 13 13:08:54 2016 -0800
Committer: Ravi Prakash 
Committed: Tue Dec 13 13:08:54 2016 -0800

--
 hadoop-mapreduce-project/conf/mapred-env.cmd | 5 -
 hadoop-mapreduce-project/conf/mapred-env.sh  | 5 -
 2 files changed, 8 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7d9006a/hadoop-mapreduce-project/conf/mapred-env.cmd
--
diff --git a/hadoop-mapreduce-project/conf/mapred-env.cmd 
b/hadoop-mapreduce-project/conf/mapred-env.cmd
index f27943f..b50ff12 100644
--- a/hadoop-mapreduce-project/conf/mapred-env.cmd
+++ b/hadoop-mapreduce-project/conf/mapred-env.cmd
@@ -14,7 +14,10 @@
 @rem See the License for the specific language governing permissions and
 @rem limitations under the License.
 
-set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+@rem when HADOOP_JOB_HISTORYSERVER_HEAPSIZE is not defined, set it
+if not defined HADOOP_JOB_HISTORYSERVER_HEAPSIZE (
+  set HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+)
 
 set HADOOP_MAPRED_ROOT_LOGGER=%HADOOP_LOGLEVEL%,RFA
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7d9006a/hadoop-mapreduce-project/conf/mapred-env.sh
--
diff --git a/hadoop-mapreduce-project/conf/mapred-env.sh 
b/hadoop-mapreduce-project/conf/mapred-env.sh
index 6be1e27..3590882 100644
--- a/hadoop-mapreduce-project/conf/mapred-env.sh
+++ b/hadoop-mapreduce-project/conf/mapred-env.sh
@@ -15,7 +15,10 @@
 
 # export JAVA_HOME=/home/y/libexec/jdk1.6.0/
 
-export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+# when HADOOP_JOB_HISTORYSERVER_HEAPSIZE is not defined, set it.
+if [ "$HADOOP_JOB_HISTORYSERVER_HEAPSIZE" = "" ];then
+  export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000
+fi
 
 export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[13/13] hadoop git commit: YARN-5587. Add support for resource profiles. (vvasudev via asuresh)

[asuresh]

YARN-5587. Add support for resource profiles. (vvasudev via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb6e4f08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb6e4f08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb6e4f08

Branch: refs/heads/YARN-5501
Commit: bb6e4f08efe44024a551dbb19b02320ca56e7ac2
Parents: 0a30b9c
Author: Arun Suresh 
Authored: Tue Nov 15 01:01:07 2016 -0800
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:33 2016 -0800

--
 .../dev-support/findbugs-exclude.xml|   4 +
 .../RegisterApplicationMasterResponse.java  |   8 +
 .../yarn/api/records/ProfileCapability.java |  94 ++-
 .../hadoop/yarn/api/records/Resource.java   |  14 ++
 .../yarn/api/records/ResourceInformation.java   |  57 ++-
 .../yarn/api/records/ResourceRequest.java   |  43 -
 .../hadoop-yarn/hadoop-yarn-client/pom.xml  |   1 +
 .../hadoop/yarn/client/api/AMRMClient.java  | 117 +-
 .../yarn/client/api/impl/AMRMClientImpl.java| 152 ++---
 .../client/api/impl/RemoteRequestsTable.java| 109 +
 .../yarn/client/api/impl/TestAMRMClient.java| 147 +++--
 .../impl/TestAMRMClientContainerRequest.java|   8 +-
 .../api/impl/TestDistributedScheduling.java |  12 +-
 .../yarn/client/api/impl/TestNMClient.java  |   5 +-
 .../TestOpportunisticContainerAllocation.java   |  15 +-
 .../src/test/resources/resource-profiles.json   |  18 +++
 ...RegisterApplicationMasterResponsePBImpl.java |  58 +++
 .../api/records/impl/pb/ResourcePBImpl.java |   4 +-
 .../records/impl/pb/ResourceRequestPBImpl.java  |  41 -
 .../yarn/util/resource/ResourceUtils.java   | 161 ++-
 .../hadoop/yarn/util/resource/Resources.java|  10 +-
 .../ApplicationMasterService.java   |  12 ++
 .../server/resourcemanager/RMServerUtils.java   |  50 ++
 .../resource/ResourceProfilesManagerImpl.java   |   4 +
 .../scheduler/AbstractYarnScheduler.java|  44 +
 .../scheduler/ClusterNodeTracker.java   |   3 +-
 .../scheduler/SchedulerUtils.java   |  10 ++
 .../scheduler/capacity/CapacityScheduler.java   |   4 +-
 .../scheduler/fair/FairScheduler.java   |   4 +-
 .../scheduler/fifo/FifoScheduler.java   |  13 +-
 .../yarn/server/resourcemanager/MockRM.java |   2 +
 .../server/resourcemanager/TestAppManager.java  |   1 +
 .../TestApplicationMasterService.java   |  35 
 .../scheduler/fair/TestFairScheduler.java   |   4 +
 .../hadoop/yarn/server/MiniYARNCluster.java |   2 +
 35 files changed, 1102 insertions(+), 164 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml 
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index ab36a4e..de58251 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -151,6 +151,10 @@
 
   
   
+
+
+  
+  
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
index 0b886dd..8fa8563 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
@@ -204,4 +204,12 @@ public abstract class RegisterApplicationMasterResponse {
   @Unstable
   public abstract void setSchedulerResourceTypes(
   EnumSet types);
+
+  @Public
+  @Unstable
+  public abstract Map getResourceProfiles();
+
+  @Private
+  @Unstable
+  public abstract void setResourceProfiles(Map profiles);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ProfileCapability.java

[07/13] hadoop git commit: YARN-4172. Extend DominantResourceCalculator to account for all resources. (Varun Vasudev via wangda)

[asuresh]

YARN-4172. Extend DominantResourceCalculator to account for all resources. 
(Varun Vasudev via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b11fa66d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b11fa66d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b11fa66d

Branch: refs/heads/YARN-5501
Commit: b11fa66d32d4f98f715b2fb56285e961962e227b
Parents: 3d32231
Author: Wangda Tan 
Authored: Fri Jan 29 10:53:31 2016 +0800
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:32 2016 -0800

--
 .../resource/DominantResourceCalculator.java| 389 +--
 1 file changed, 273 insertions(+), 116 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b11fa66d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 9f1c8d7..4a8eeb2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -22,59 +22,103 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+
+import java.util.HashSet;
+import java.util.Set;
 
 /**
- * A {@link ResourceCalculator} which uses the concept of  
+ * A {@link ResourceCalculator} which uses the concept of
  * dominant resource to compare multi-dimensional resources.
  *
- * Essentially the idea is that the in a multi-resource environment, 
- * the resource allocation should be determined by the dominant share 
- * of an entity (user or queue), which is the maximum share that the 
- * entity has been allocated of any resource. 
- * 
- * In a nutshell, it seeks to maximize the minimum dominant share across 
- * all entities. 
- * 
+ * Essentially the idea is that the in a multi-resource environment,
+ * the resource allocation should be determined by the dominant share
+ * of an entity (user or queue), which is the maximum share that the
+ * entity has been allocated of any resource.
+ *
+ * In a nutshell, it seeks to maximize the minimum dominant share across
+ * all entities.
+ *
  * For example, if user A runs CPU-heavy tasks and user B runs
- * memory-heavy tasks, it attempts to equalize CPU share of user A 
- * with Memory-share of user B. 
- * 
+ * memory-heavy tasks, it attempts to equalize CPU share of user A
+ * with Memory-share of user B.
+ *
  * In the single resource case, it reduces to max-min fairness for that 
resource.
- * 
+ *
  * See the Dominant Resource Fairness paper for more details:
  * www.cs.berkeley.edu/~matei/papers/2011/nsdi_drf.pdf
  */
 @Private
 @Unstable
 public class DominantResourceCalculator extends ResourceCalculator {
+
   private static final Log LOG =
   LogFactory.getLog(DominantResourceCalculator.class);
 
+  private Set resourceNames;
+
+  public DominantResourceCalculator() {
+resourceNames = new HashSet<>();
+resourceNames.add(ResourceInformation.MEMORY.getName());
+resourceNames.add(ResourceInformation.VCORES.getName());
+  }
+
+  /**
+   * Compare two resources - if the value for every resource type for the lhs
+   * is greater than that of the rhs, return 1. If the value for every resource
+   * type in the lhs is less than the rhs, return -1. Otherwise, return 0
+   *
+   * @param lhs resource to be compared
+   * @param rhs resource to be compared
+   * @return 0, 1, or -1
+   */
+  private int compare(Resource lhs, Resource rhs) {
+boolean lhsGreater = false;
+boolean rhsGreater = false;
+int ret = 0;
+
+for (String rName : resourceNames) {
+  try {
+ResourceInformation lhsResourceInformation =
+lhs.getResourceInformation(rName);
+ResourceInformation rhsResourceInformation =
+rhs.getResourceInformation(rName);
+int diff = lhsResourceInformation.compareTo(rhsResourceInformation);
+if (diff >= 1) {
+  lhsGreater = true;
+

[09/13] hadoop git commit: YARN-5242. Update DominantResourceCalculator to consider all resource types in calculations. Contributed by Varun Vasudev.

[asuresh]

YARN-5242. Update DominantResourceCalculator to consider all resource types in 
calculations. Contributed by Varun Vasudev.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9f7c10f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9f7c10f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9f7c10f

Branch: refs/heads/YARN-5501
Commit: f9f7c10f74e856f440af9bb544fcb3412bb69353
Parents: f079243
Author: Rohith Sharma K S 
Authored: Tue Jul 26 14:13:03 2016 +0530
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:32 2016 -0800

--
 .../hadoop/yarn/api/records/Resource.java   |  7 ++
 .../api/records/impl/pb/ResourcePBImpl.java |  2 +-
 .../resource/DominantResourceCalculator.java| 23 
 .../yarn/util/resource/ResourceUtils.java   |  5 +++--
 .../hadoop/yarn/util/resource/Resources.java|  6 +
 5 files changed, 31 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9f7c10f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 3880bb0..4bbca5a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -291,6 +291,8 @@ public abstract class Resource implements 
Comparable {
 otherResources = other.getResources();
 long diff = thisResources.size() - otherResources.size();
 if (diff == 0) {
+  // compare memory and vcores first(in that order) to preserve
+  // existing behaviour
   if (thisResources.keySet().equals(otherResources.keySet())) {
 diff = this.getMemorySize() - other.getMemorySize();
 if (diff == 0) {
@@ -299,6 +301,11 @@ public abstract class Resource implements 
Comparable {
 if (diff == 0) {
   for (Map.Entry entry : thisResources
   .entrySet()) {
+if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
+|| entry.getKey()
+.equals(ResourceInformation.VCORES.getName())) {
+  continue;
+}
 diff =
 entry.getValue().compareTo(otherResources.get(entry.getKey()));
 if (diff != 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9f7c10f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 8cf7291..36e5beb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -228,7 +228,7 @@ public class ResourcePBImpl extends Resource {
 builder.addResourceValueMap(e);
   }
 }
-builder.setMemory(this.getMemory());
+builder.setMemory(this.getMemorySize());
 builder.setVirtualCores(this.getVirtualCores());
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9f7c10f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index fc078e4..f9386b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -387,9 +387,24 @@ public class DominantResourceCalculator extends 

[02/13] hadoop git commit: YARN-4830. Add support for resource types in the nodemanager. Contributed by Varun Vasudev.

[asuresh]

YARN-4830. Add support for resource types in the nodemanager. Contributed by 
Varun Vasudev.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0792432
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0792432
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0792432

Branch: refs/heads/YARN-5501
Commit: f07924329227815e66f306a4e690970cccba1e9f
Parents: 04536d8
Author: Varun Vasudev 
Authored: Sat Jun 11 14:33:46 2016 +0530
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:32 2016 -0800

--
 .../hadoop/yarn/api/records/Resource.java   |   3 +-
 .../hadoop/yarn/conf/YarnConfiguration.java |  19 ++-
 .../FileSystemBasedConfigurationProvider.java   |   3 +-
 .../hadoop/yarn/LocalConfigurationProvider.java |   3 +-
 .../api/records/impl/pb/ResourcePBImpl.java |  53 +++---
 .../yarn/util/resource/ResourceUtils.java   | 168 +++
 .../yarn/util/resource/TestResourceUtils.java   |  29 +++-
 .../resource-types/node-resources-1.xml |  29 
 .../resource-types/node-resources-2.xml |  39 +
 .../nodemanager/NodeStatusUpdaterImpl.java  |   7 +-
 .../util/NodeManagerHardwareUtils.java  |  52 ++
 .../resourcemanager/ResourceTrackerService.java |   9 +-
 12 files changed, 342 insertions(+), 72 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0792432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 3efccce..3880bb0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -272,7 +272,8 @@ public abstract class Resource implements 
Comparable {
 continue;
   }
   if (entry.getKey().equals(ResourceInformation.VCORES.getName())
-  && entry.getValue().getUnits().equals("")) {
+  && entry.getValue().getUnits()
+  .equals(ResourceInformation.VCORES.getUnits())) {
 continue;
   }
   sb.append(", ").append(entry.getKey()).append(": ")

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0792432/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 563778a..888ff83 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -63,6 +63,10 @@ public class YarnConfiguration extends Configuration {
   "resource-types.xml";
 
   @Private
+  public static final String NODE_RESOURCES_CONFIGURATION_FILE =
+  "node-resources.xml";
+
+  @Private
   public static final List RM_CONFIGURATION_FILES =
   Collections.unmodifiableList(Arrays.asList(
   RESOURCE_TYPES_CONFIGURATION_FILE,
@@ -72,6 +76,16 @@ public class YarnConfiguration extends Configuration {
   YARN_SITE_CONFIGURATION_FILE,
   CORE_SITE_CONFIGURATION_FILE));
 
+  @Private
+  public static final List NM_CONFIGURATION_FILES =
+  Collections.unmodifiableList(Arrays.asList(
+  NODE_RESOURCES_CONFIGURATION_FILE,
+  DR_CONFIGURATION_FILE,
+  CS_CONFIGURATION_FILE,
+  HADOOP_POLICY_CONFIGURATION_FILE,
+  YARN_SITE_CONFIGURATION_FILE,
+  CORE_SITE_CONFIGURATION_FILE));
+
   @Evolving
   public static final int APPLICATION_MAX_TAGS = 10;
 
@@ -100,12 +114,15 @@ public class YarnConfiguration extends Configuration {
   public static final String YARN_PREFIX = "yarn.";
 
   /
-  // Scheduler resource types configs
+  // Resource types configs
   
 
   public static final String RESOURCE_TYPES =
   YarnConfiguration.YARN_PREFIX + "resource-types";
 
+  public static final String NM_RESOURCES_PREFIX =
+  YarnConfiguration.NM_PREFIX + "resource-type.";
+
   /** Delay before deleting resource to ease debugging of NM issues 

[12/13] hadoop git commit: YARN-5587. Add support for resource profiles. (vvasudev via asuresh)

[asuresh]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
index 9890296..39d772a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java
@@ -23,8 +23,10 @@ import 
org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
 import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ProfileCapability;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.proto.YarnProtos.ProfileCapabilityProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
@@ -40,6 +42,7 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   private Priority priority = null;
   private Resource capability = null;
   private ExecutionTypeRequest executionTypeRequest = null;
+  private ProfileCapability profile = null;
   
   
   public ResourceRequestPBImpl() {
@@ -52,7 +55,7 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   }
   
   public ResourceRequestProto getProto() {
-  mergeLocalToProto();
+mergeLocalToProto();
 proto = viaProto ? proto : builder.build();
 viaProto = true;
 return proto;
@@ -69,6 +72,9 @@ public class ResourceRequestPBImpl extends  ResourceRequest {
   builder.setExecutionTypeRequest(
   ProtoUtils.convertToProtoFormat(this.executionTypeRequest));
 }
+if (this.profile != null) {
+  builder.setProfile(converToProtoFormat(this.profile));
+}
   }
 
   private void mergeLocalToProto() {
@@ -229,7 +235,8 @@ public class ResourceRequestPBImpl extends  ResourceRequest 
{
 + ", Location: " + getResourceName()
 + ", Relax Locality: " + getRelaxLocality()
 + ", Execution Type Request: " + getExecutionTypeRequest()
-+ ", Node Label Expression: " + getNodeLabelExpression() + "}";
++ ", Node Label Expression: " + getNodeLabelExpression()
++ ", Resource Profile: " + getProfileCapability() + "}";
   }
 
   @Override
@@ -250,4 +257,34 @@ public class ResourceRequestPBImpl extends  
ResourceRequest {
 }
 builder.setNodeLabelExpression(nodeLabelExpression);
   }
+
+  @Override
+  public void setProfileCapability(ProfileCapability profileCapability) {
+maybeInitBuilder();
+if (profile == null) {
+  builder.clearProfile();
+}
+this.profile = profileCapability;
+  }
+
+  @Override
+  public ProfileCapability getProfileCapability() {
+if (profile != null) {
+  return profile;
+}
+ResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
+if (!p.hasProfile()) {
+  return null;
+}
+return new ProfileCapabilityPBImpl(p.getProfile());
+  }
+
+  private ProfileCapabilityProto converToProtoFormat(
+  ProfileCapability profileCapability) {
+ProfileCapabilityPBImpl tmp = new ProfileCapabilityPBImpl();
+tmp.setProfileName(profileCapability.getProfileName());
+tmp.setProfileCapabilityOverride(
+profileCapability.getProfileCapabilityOverride());
+return tmp.getProto();
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb6e4f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 938e462..86cf872 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
 import 

[11/13] hadoop git commit: YARN-5708. Implement APIs to get resource profiles from the RM. Contributed by Varun Vasudev.

[asuresh]

YARN-5708. Implement APIs to get resource profiles from the RM. Contributed by 
Varun Vasudev.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a30b9c4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a30b9c4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a30b9c4

Branch: refs/heads/YARN-5501
Commit: 0a30b9c406f38d71a84bca1efb123053a077edae
Parents: 8699cc6
Author: Varun Vasudev 
Authored: Sat Oct 22 20:15:47 2016 +0530
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:33 2016 -0800

--
 .../hadoop/mapred/ResourceMgrDelegate.java  |  13 ++
 .../hadoop/mapred/TestClientRedirect.java   |  17 +++
 .../yarn/api/ApplicationClientProtocol.java |  37 +
 .../GetAllResourceProfilesRequest.java  |  35 +
 .../GetAllResourceProfilesResponse.java |  60 
 .../GetResourceProfileRequest.java  |  59 
 .../GetResourceProfileResponse.java |  68 +
 .../yarn/api/records/ProfileCapability.java |  88 
 .../main/proto/applicationclient_protocol.proto |   2 +
 .../src/main/proto/yarn_protos.proto|  15 ++
 .../src/main/proto/yarn_service_protos.proto|  16 +++
 .../hadoop/yarn/client/api/YarnClient.java  |  25 
 .../yarn/client/api/impl/YarnClientImpl.java|  18 +++
 .../ApplicationClientProtocolPBClientImpl.java  |  36 +
 .../ApplicationClientProtocolPBServiceImpl.java |  42 ++
 .../pb/GetAllResourceProfilesRequestPBImpl.java |  55 +++
 .../GetAllResourceProfilesResponsePBImpl.java   | 142 +++
 .../pb/GetResourceProfileRequestPBImpl.java | 101 +
 .../pb/GetResourceProfileResponsePBImpl.java| 112 +++
 .../impl/pb/ProfileCapabilityPBImpl.java| 134 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |  34 +
 .../amrmproxy/MockResourceManagerFacade.java|  16 +++
 .../server/resourcemanager/ClientRMService.java |  41 ++
 .../yarn/server/resourcemanager/RMContext.java  |   5 +
 .../server/resourcemanager/RMContextImpl.java   |  15 ++
 .../server/resourcemanager/ResourceManager.java |   8 ++
 26 files changed, 1194 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a30b9c4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index c302553..6ed0127 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
@@ -517,4 +518,16 @@ public class ResourceMgrDelegate extends YarnClient {
   throws YarnException, IOException {
 client.killApplication(appId, diagnostics);
   }
+
+  @Override
+  public Map getResourceProfiles()
+  throws YarnException, IOException {
+return client.getResourceProfiles();
+  }
+
+  @Override
+  public Resource getResourceProfile(String profile)
+  throws YarnException, IOException {
+return client.getResourceProfile(profile);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a30b9c4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index 65eac65..cc50be0 100644
--- 

[08/13] hadoop git commit: YARN-5586. Update the Resources class to consider all resource types. Contributed by Varun Vasudev.

[asuresh]

YARN-5586. Update the Resources class to consider all resource types. 
Contributed by Varun Vasudev.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66c487cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66c487cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66c487cd

Branch: refs/heads/YARN-5501
Commit: 66c487cdd64ddba2ac824bdcce355fae093a7ec5
Parents: f9f7c10
Author: Rohith Sharma K S 
Authored: Mon Sep 12 10:44:26 2016 +0530
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:32 2016 -0800

--
 .../api/records/impl/pb/ResourcePBImpl.java |   4 +-
 .../resource/DominantResourceCalculator.java|  36 ++--
 .../yarn/util/resource/ResourceUtils.java   |   3 +-
 .../hadoop/yarn/util/resource/Resources.java| 138 ++--
 .../yarn/util/resource/TestResourceUtils.java   |  23 ++
 .../yarn/util/resource/TestResources.java   | 212 +--
 .../resourcemanager/resource/TestResources.java |  43 
 7 files changed, 369 insertions(+), 90 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/66c487cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 36e5beb..236a763 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -156,7 +156,9 @@ public class ResourcePBImpl extends Resource {
   resourceInformation.setName(resource);
 }
 initResources();
-resources.put(resource, resourceInformation);
+if (resources.containsKey(resource)) {
+  resources.put(resource, resourceInformation);
+}
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/66c487cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index f9386b5..19c9654 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -181,8 +181,10 @@ public class DominantResourceCalculator extends 
ResourceCalculator {
 Long requiredResourceValue = UnitsConversionUtil
 .convert(requiredResource.getUnits(), availableResource.getUnits(),
 requiredResource.getValue());
-Long tmp = availableResource.getValue() / requiredResourceValue;
-min = min < tmp ? min : tmp;
+if (requiredResourceValue != 0) {
+  Long tmp = availableResource.getValue() / requiredResourceValue;
+  min = min < tmp ? min : tmp;
+}
   } catch (YarnException ye) {
 throw new IllegalArgumentException(
 "Error getting resource information for " + resource, ye);
@@ -291,10 +293,11 @@ public class DominantResourceCalculator extends 
ResourceCalculator {
 .convert(stepFactorResourceInformation.getUnits(),
 rResourceInformation.getUnits(),
 stepFactorResourceInformation.getValue());
-
-tmp.setValue(
-Math.min(roundUp(Math.max(rValue, minimumValue), stepFactorValue),
-maximumValue));
+Long value = Math.max(rValue, minimumValue);
+if (stepFactorValue != 0) {
+  value = roundUp(value, stepFactorValue);
+}
+tmp.setValue(Math.min(value, maximumValue));
 ret.setResourceInformation(resource, tmp);
   } catch (YarnException ye) {
 throw new IllegalArgumentException(
@@ -330,9 +333,11 @@ public class DominantResourceCalculator extends 
ResourceCalculator {
 .convert(stepFactorResourceInformation.getUnits(),
 rResourceInformation.getUnits(),
 

[03/13] hadoop git commit: YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)

[asuresh]

YARN-4829. Add support for binary units in Resource class.(vvasudev via asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/04536d84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/04536d84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/04536d84

Branch: refs/heads/YARN-5501
Commit: 04536d84e34fd14e8a2c3da48f027be3d102b1c3
Parents: e439c47
Author: Arun Suresh 
Authored: Thu Mar 17 23:50:22 2016 -0700
Committer: Arun Suresh 
Committed: Tue Dec 13 12:10:32 2016 -0800

--
 .../yarn/api/records/ResourceInformation.java   |  2 +-
 .../hadoop/yarn/util/UnitsConversionUtil.java   | 45 ++--
 .../yarn/util/TestUnitsConversionUtil.java  | 17 +++-
 .../api/records/impl/pb/ResourcePBImpl.java |  2 +-
 .../yarn/util/resource/TestResourceUtils.java   |  2 +-
 5 files changed, 52 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/04536d84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 80e3192..a17e81b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -36,7 +36,7 @@ public class ResourceInformation implements 
Comparable {
   private static final String VCORES_URI = "vcores";
 
   public static final ResourceInformation MEMORY_MB =
-  ResourceInformation.newInstance(MEMORY_URI, "M");
+  ResourceInformation.newInstance(MEMORY_URI, "Mi");
   public static final ResourceInformation VCORES =
   ResourceInformation.newInstance(VCORES_URI);
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/04536d84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
index 7785263..47bb3df 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
@@ -46,7 +46,8 @@ public class UnitsConversionUtil {
   }
 
   private static final String[] UNITS =
-  {"p", "n", "u", "m", "", "k", "M", "G", "T", "P"};
+  { "p", "n", "u", "m", "", "k", "M", "G", "T", "P", "Ki", "Mi", "Gi", 
"Ti",
+  "Pi" };
   private static final List SORTED_UNITS = Arrays.asList(UNITS);
   public static final Set KNOWN_UNITS = createKnownUnitsSet();
   private static final Converter PICO =
@@ -65,6 +66,15 @@ public class UnitsConversionUtil {
   private static final Converter PETA =
   new Converter(1000L * 1000L * 1000L * 1000L * 1000L, 1L);
 
+  private static final Converter KILO_BINARY = new Converter(1024L, 1L);
+  private static final Converter MEGA_BINARY = new Converter(1024L * 1024L, 
1L);
+  private static final Converter GIGA_BINARY =
+  new Converter(1024L * 1024L * 1024L, 1L);
+  private static final Converter TERA_BINARY =
+  new Converter(1024L * 1024L * 1024L * 1024L, 1L);
+  private static final Converter PETA_BINARY =
+  new Converter(1024L * 1024L * 1024L * 1024L * 1024L, 1L);
+
   private static Set createKnownUnitsSet() {
 Set ret = new HashSet<>();
 ret.addAll(Arrays.asList(UNITS));
@@ -93,6 +103,16 @@ public class UnitsConversionUtil {
   return TERA;
 case "P":
   return PETA;
+case "Ki":
+  return KILO_BINARY;
+case "Mi":
+  return MEGA_BINARY;
+case "Gi":
+  return GIGA_BINARY;
+case "Ti":
+  return TERA_BINARY;
+case "Pi":
+  return PETA_BINARY;
 default:
   throw new IllegalArgumentException(
   "Unknown unit '" + unit + "'. Known units are " + KNOWN_UNITS);
@@ -112,28 +132,29 @@ public class UnitsConversionUtil {
 if (toUnit == null || fromUnit == null || fromValue == null) {
   throw new IllegalArgumentException("One or more arguments are null");
 }
-Long tmp;
 String overflowMsg =