hadoop git commit: YARN-6982. Potential issue on setting AMContainerSpec#tokenConf to null before app is completed. Contributed by Manikandan R.

[rohithsharmaks]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 870ef0b2f -> 877d96b83


YARN-6982. Potential issue on setting AMContainerSpec#tokenConf to null before 
app is completed. Contributed by Manikandan R.

(cherry picked from commit 4cae120c619811006b26b9a95680a98732572af6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/877d96b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/877d96b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/877d96b8

Branch: refs/heads/branch-2
Commit: 877d96b83d67a4d1c4ccdd4a5333f4ade483d8ec
Parents: 870ef0b
Author: Rohith Sharma K S 
Authored: Wed Aug 30 10:45:11 2017 +0530
Committer: Rohith Sharma K S 
Committed: Wed Aug 30 10:50:57 2017 +0530

--
 .../server/resourcemanager/rmapp/RMAppImpl.java |   4 -
 .../yarn/server/resourcemanager/MockRM.java |  11 ++
 .../server/resourcemanager/TestRMRestart.java   | 115 +++
 3 files changed, 126 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/877d96b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 181c5f4..a9351f7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1035,8 +1035,6 @@ public class RMAppImpl implements RMApp, Recoverable {
   app.submissionContext.getCancelTokensWhenComplete(),
   app.getUser(),
   BuilderUtils.parseTokensConf(app.submissionContext));
-  // set the memory free
-  app.submissionContext.getAMContainerSpec().setTokensConf(null);
 } catch (Exception e) {
   String msg = "Failed to fetch user credentials from application:"
   + e.getMessage();
@@ -1089,8 +1087,6 @@ public class RMAppImpl implements RMApp, Recoverable {
   app.submissionContext, false, app.applicationPriority));
   // send the ATS create Event
   app.sendATSCreateEvent();
-  // Set the memory free after submission context is persisted
-  app.submissionContext.getAMContainerSpec().setTokensConf(null);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/877d96b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 98e96f5..90507f3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -513,6 +513,17 @@ public class MockRM extends ResourceManager {
 false, false, null, 0, null, true, priority);
   }
 
+  public RMApp submitApp(int masterMemory, Priority priority,
+  Credentials cred, ByteBuffer tokensConf) throws Exception {
+Resource resource = Resource.newInstance(masterMemory, 0);
+return submitApp(resource, "", UserGroupInformation.getCurrentUser()
+.getShortUserName(), null, false, null,
+super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true,
+false, false, null, 0, null, true, priority, null, null,
+tokensConf);
+  }
+
   public RMApp submitApp(int masterMemory, boolean unmanaged)
   throws Exception {
 return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()

hadoop git commit: YARN-6982. Potential issue on setting AMContainerSpec#tokenConf to null before app is completed. Contributed by Manikandan R.

[rohithsharmaks]

Repository: hadoop
Updated Branches:
  refs/heads/trunk f3661fd08 -> 4cae120c6


YARN-6982. Potential issue on setting AMContainerSpec#tokenConf to null before 
app is completed. Contributed by Manikandan R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cae120c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cae120c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cae120c

Branch: refs/heads/trunk
Commit: 4cae120c619811006b26b9a95680a98732572af6
Parents: f3661fd
Author: Rohith Sharma K S 
Authored: Wed Aug 30 10:45:11 2017 +0530
Committer: Rohith Sharma K S 
Committed: Wed Aug 30 10:45:11 2017 +0530

--
 .../server/resourcemanager/rmapp/RMAppImpl.java |   4 -
 .../yarn/server/resourcemanager/MockRM.java |  11 ++
 .../server/resourcemanager/TestRMRestart.java   | 115 +++
 3 files changed, 126 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cae120c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 03be793..af7cec4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1086,8 +1086,6 @@ public class RMAppImpl implements RMApp, Recoverable {
   app.submissionContext.getCancelTokensWhenComplete(),
   app.getUser(),
   BuilderUtils.parseTokensConf(app.submissionContext));
-  // set the memory free
-  app.submissionContext.getAMContainerSpec().setTokensConf(null);
 } catch (Exception e) {
   String msg = "Failed to fetch user credentials from application:"
   + e.getMessage();
@@ -1140,8 +1138,6 @@ public class RMAppImpl implements RMApp, Recoverable {
   app.submissionContext, false, app.applicationPriority));
   // send the ATS create Event
   app.sendATSCreateEvent();
-  // Set the memory free after submission context is persisted
-  app.submissionContext.getAMContainerSpec().setTokensConf(null);
 }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cae120c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index e967807..1235774 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -513,6 +513,17 @@ public class MockRM extends ResourceManager {
 false, false, null, 0, null, true, priority);
   }
 
+  public RMApp submitApp(int masterMemory, Priority priority,
+  Credentials cred, ByteBuffer tokensConf) throws Exception {
+Resource resource = Resource.newInstance(masterMemory, 0);
+return submitApp(resource, "", UserGroupInformation.getCurrentUser()
+.getShortUserName(), null, false, null,
+super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), cred, null, true,
+false, false, null, 0, null, true, priority, null, null,
+tokensConf);
+  }
+
   public RMApp submitApp(int masterMemory, boolean unmanaged)
   throws Exception {
 return submitApp(masterMemory, "", UserGroupInformation.getCurrentUser()

hadoop git commit: HDFS-12191. Provide option to not capture the accessTime change of a file to snapshot if no other modification has been done to this file. Contributed by Yongjun Zhang.

[yjzhangal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 036cbd745 -> 870ef0b2f


HDFS-12191. Provide option to not capture the accessTime change of a file to 
snapshot if no other modification has been done to this file. Contributed by 
Yongjun Zhang.

(cherry picked from commit cf93d60d3f032000e5b78a08d320793d78799f3d)

Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirAttrOp.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/870ef0b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/870ef0b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/870ef0b2

Branch: refs/heads/branch-2
Commit: 870ef0b2fd26c9f748e9b309cf1a5235e31fc8ab
Parents: 036cbd7
Author: Yongjun Zhang 
Authored: Tue Aug 29 16:57:03 2017 -0700
Committer: Yongjun Zhang 
Committed: Tue Aug 29 21:57:25 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   3 +
 .../hdfs/server/namenode/FSDirAttrOp.java   |   4 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   4 +-
 .../hadoop/hdfs/server/namenode/INode.java  |   7 +-
 .../namenode/snapshot/SnapshotManager.java  |  28 
 .../src/main/resources/hdfs-default.xml |  44 +++--
 .../hdfs/server/namenode/TestFSDirAttrOp.java   |   7 +
 .../snapshot/TestSnapshotDiffReport.java| 163 ++-
 8 files changed, 233 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/870ef0b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index d5192c2..185c882 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -331,6 +331,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
   HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT;
 
+  public static final String 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE = 
"dfs.namenode.snapshot.skip.capture.accesstime-only-change";
+  public static final boolean 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE_DEFAULT = false;
+
   // Whether to enable datanode's stale state detection and usage for reads
   public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = 
"dfs.namenode.avoid.read.stale.datanode";
   public static final boolean 
DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/870ef0b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 9e714af..478d5ac 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -493,7 +493,9 @@ public class FSDirAttrOp {
 // then no need to store access time
 if (atime != -1 && (status || force
 || atime > inode.getAccessTime() + fsd.getAccessTimePrecision())) {
-  inode.setAccessTime(atime, latest);
+  inode.setAccessTime(atime, latest,
+  fsd.getFSNamesystem().getSnapshotManager().
+  getSkipCaptureAccessTimeOnlyChange());
   status = true;
 }
 return status;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/870ef0b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 616c2e3..d603559 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ 

hadoop git commit: HDFS-12336. Listing encryption zones still fails when deleted EZ is not a direct child of snapshottable directory. Contributed by Wellington Chevreuil.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 8201ed800 -> f3661fd08


HDFS-12336. Listing encryption zones still fails when deleted EZ is not a 
direct child of snapshottable directory. Contributed by Wellington Chevreuil.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3661fd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3661fd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3661fd0

Branch: refs/heads/trunk
Commit: f3661fd08e03440d02452b46ba3ae3cff2b75ba7
Parents: 8201ed8
Author: Xiao Chen 
Authored: Tue Aug 29 21:28:02 2017 -0700
Committer: Xiao Chen 
Committed: Tue Aug 29 21:30:02 2017 -0700

--
 .../server/namenode/EncryptionZoneManager.java  |  2 +-
 .../hadoop/hdfs/server/namenode/INode.java  | 11 -
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 47 
 .../namenode/TestEncryptionZoneManager.java | 26 +++
 .../src/test/resources/testCryptoConf.xml   | 32 +
 5 files changed, 116 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3661fd0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index d6302ba..f4cf8f2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -573,7 +573,7 @@ public class EncryptionZoneManager {
   return false;
 }
 INode lastINode = null;
-if (inode.getParent() != null || inode.isRoot()) {
+if (INode.isValidAbsolutePath(zonePath)) {
   INodesInPath iip = dir.getINodesInPath(zonePath, DirOp.READ_LINK);
   lastINode = iip.getLastINode();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3661fd0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index 874563d..34bfe10 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -783,8 +783,17 @@ public abstract class INode implements INodeAttributes, 
Diff.Element {
 return StringUtils.split(path, Path.SEPARATOR_CHAR);
   }
 
+  /**
+   * Verifies if the path informed is a valid absolute path.
+   * @param path the absolute path to validate.
+   * @return true if the path is valid.
+   */
+  static boolean isValidAbsolutePath(final String path){
+return path != null && path.startsWith(Path.SEPARATOR);
+  }
+
   private static void checkAbsolutePath(final String path) {
-if (path == null || !path.startsWith(Path.SEPARATOR)) {
+if (!isValidAbsolutePath(path)) {
   throw new AssertionError("Absolute path required, but got '"
   + path + "'");
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3661fd0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index bf02db3..870023b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1867,4 +1867,51 @@ public class TestEncryptionZones {
 // Read them back in and compare byte-by-byte
 verifyFilesEqual(fs, baseFile, encFile1, len);
   }
+
+  /**
+   * Test listing encryption zones after zones had been deleted,
+   * but still exist under snapshots. This test first moves EZs
+   * to trash folder, so that an inodereference is created for the EZ,
+   * then it removes the EZ from trash folder to emulate condition where
+   * the EZ inode will not be complete.
+   */
+  @Test
+  public void testListEncryptionZonesWithSnapshots() throws 

[46/75] [abbrv] hadoop git commit: YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang

[jianhe]

YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by 
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e7ddba0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e7ddba0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e7ddba0

Branch: refs/heads/yarn-native-services
Commit: 6e7ddba0c5e80b509f708167b40d547eb4504d00
Parents: ef230a3
Author: Jian He 
Authored: Tue Aug 29 20:37:38 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:45:12 2017 -0700

--
 .../java/org/apache/hadoop/yarn/service/api/records/Resource.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e7ddba0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
index cec9de9..dfdf92a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty;
 
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -103,6 +104,7 @@ public class Resource extends BaseResource implements 
Cloneable {
 this.memory = memory;
   }
 
+  @JsonIgnore
   public long getMemoryMB() {
 if (this.memory == null) {
   return 0;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[22/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
deleted file mode 100644
index d96d13e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.hadoop.yarn.service.provider.ProviderService;
-import org.apache.hadoop.yarn.service.provider.ProviderFactory;
-import org.apache.slider.server.appmaster.actions.QueueAccess;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.services.workflow.ServiceThreadFactory;
-import org.apache.slider.server.services.workflow.WorkflowExecutorService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static 
org.apache.hadoop.yarn.service.conf.SliderKeys.KEY_CONTAINER_LAUNCH_DELAY;
-
-/**
- * A service for launching containers
- */
-public class RoleLaunchService
-extends WorkflowExecutorService {
-  protected static final Logger log =
-LoggerFactory.getLogger(RoleLaunchService.class);
-
-  public static final String ROLE_LAUNCH_SERVICE = "RoleLaunchService";
-
-
-  /**
-   * Queue submission API
-   */
-  private  QueueAccess actionQueue;
-
-  /**
-   * Filesystem to use for the launch
-   */
-  private  SliderFileSystem fs;
-
-
-  private Map envVars = new HashMap<>();
-
-  /**
-   * Construct an instance of the launcher
-   * @param queueAccess
-   * @param fs filesystem
-   * @param envVars environment variables
-   */
-  public RoleLaunchService(QueueAccess queueAccess, SliderFileSystem fs,
-  Map envVars) {
-super(ROLE_LAUNCH_SERVICE);
-this.actionQueue = queueAccess;
-this.fs = fs;
-this.envVars = envVars;
-  }
-
-  public RoleLaunchService(SliderFileSystem fs) {
-super(ROLE_LAUNCH_SERVICE);
-this.fs = fs;
-  }
-
-  @Override
-  public void init(Configuration conf) {
-super.init(conf);
-setExecutor(Executors.newCachedThreadPool(
-new ServiceThreadFactory(ROLE_LAUNCH_SERVICE, true)));
-  }
-
-  /**
-   * Start an asychronous launch operation
-   * @param assignment container assignment
-   * @param credentials credentials to use
-   */
-  public void launchRole(ContainerAssignment assignment,
-  Application application, Credentials credentials) {
-  }
-
-  public void launchComponent(Application application,
-  ComponentInstance instance, Container container) {
-RoleLaunchService.RoleLauncher launcher =
-new RoleLaunchService.RoleLauncher(application, instance,
-container);
-execute(launcher);
-  }
-
-  /**
-   * Thread that runs on the AM to launch a container
-   */
-  private class RoleLauncher implements Runnable {
-// Allocated container
-public final Container container;
-public final Application application;
- 

[66/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
new file mode 100644
index 000..7baa284
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Build a single command line to include in the container commands;
+ * Special support for JVM command buildup.
+ */
+public class CommandLineBuilder {
+  protected final List argumentList = new ArrayList<>(20);
+
+  /**
+   * Add an entry to the command list
+   * @param args arguments -these will be converted strings
+   */
+  public void add(Object... args) {
+for (Object arg : args) {
+  argumentList.add(arg.toString());
+}
+  }
+
+  // Get the number of arguments
+  public int size() {
+return argumentList.size();
+  }
+  
+  /**
+   * Append the output and error files to the tail of the command
+   * @param stdout out
+   * @param stderr error. Set this to null to append into stdout
+   */
+  public void addOutAndErrFiles(String stdout, String stderr) {
+Preconditions.checkNotNull(stdout, "Null output file");
+Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
+// write out the path output
+argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
+ stdout);
+if (stderr != null) {
+  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
+   stderr);
+} else {
+  argumentList.add("2>&1");
+}
+  }
+
+  /**
+   * This just returns the command line
+   * @see #build()
+   * @return the command line
+   */
+  @Override
+  public String toString() {
+return build();
+  }
+
+  /**
+   * Build the command line
+   * @return the command line
+   */
+  public String build() {
+return SliderUtils.join(argumentList, " ");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
new file mode 100644
index 000..0e51a62
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 

[08/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
deleted file mode 100644
index 057f6c5..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.common.tools;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/** Test slider util methods. */
-public class TestSliderUtils {
-  protected static final Logger log =
-  LoggerFactory.getLogger(TestSliderUtils.class);
-  @Rule
-  public TemporaryFolder folder = new TemporaryFolder();
-
-  //@Test
-  public void testTruncate() {
-Assert.assertEquals(SliderUtils.truncate(null, 5), null);
-Assert.assertEquals(SliderUtils.truncate("323", -1), "323");
-Assert.assertEquals(SliderUtils.truncate("3232", 5), "3232");
-Assert.assertEquals(SliderUtils.truncate("1234567890", 0), "1234567890");
-Assert.assertEquals(SliderUtils.truncate("123456789012345", 15), 
"123456789012345");
-Assert.assertEquals(SliderUtils.truncate("123456789012345", 14), 
"12345678901...");
-Assert.assertEquals(SliderUtils.truncate("1234567890", 1), "1");
-Assert.assertEquals(SliderUtils.truncate("1234567890", 10), "1234567890");
-Assert.assertEquals(SliderUtils.truncate("", 10), "");
-  }
-
-  //@Test
-  public void testApplicationReportComparison() {
-List instances = getApplicationReports();
-
-SliderUtils.sortApplicationsByMostRecent(instances);
-
-Assert.assertEquals(1000, instances.get(0).getStartTime());
-Assert.assertEquals(1000, instances.get(1).getStartTime());
-Assert.assertEquals(1000, instances.get(2).getStartTime());
-Assert.assertEquals(1000, instances.get(3).getStartTime());
-
-instances = getApplicationReports();
-
-SliderUtils.sortApplicationReport(instances);
-Assert.assertEquals(1000, instances.get(0).getStartTime());
-Assert.assertEquals(1000, instances.get(1).getStartTime());
-Assert.assertEquals(1000, instances.get(2).getStartTime());
-Assert.assertEquals(1000, instances.get(3).getStartTime());
-
-Assert.assertTrue(instances.get(0).getYarnApplicationState() == 
YarnApplicationState.ACCEPTED ||
-  instances.get(0).getYarnApplicationState() == 
YarnApplicationState.RUNNING);
-Assert.assertTrue(instances.get(1).getYarnApplicationState() == 
YarnApplicationState.ACCEPTED ||
-  instances.get(1).getYarnApplicationState() == 
YarnApplicationState.RUNNING);
-Assert.assertTrue(instances.get(2).getYarnApplicationState() == 
YarnApplicationState.ACCEPTED ||
-  instances.get(2).getYarnApplicationState() == 
YarnApplicationState.RUNNING);
-Assert.assertTrue(instances.get(3).getYarnApplicationState() == 
YarnApplicationState.KILLED);
-  }
-
-  private List getApplicationReports() {
-List instances = new ArrayList();
-instances.add(getApplicationReport(1000, 0, "app1", 
YarnApplicationState.ACCEPTED));
-instances.add(getApplicationReport(900, 998, "app1", 

[15/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
deleted file mode 100644
index 15333e4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ /dev/null
@@ -1,1123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.BoolMetric;
-import org.apache.slider.server.appmaster.management.Timestamp;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.slider.server.avro.LoadedRoleHistory;
-import org.apache.slider.server.avro.NodeEntryRecord;
-import org.apache.slider.server.avro.RoleHistoryHeader;
-import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * The Role History.
- * 
- * Synchronization policy: all public operations are synchronized.
- * Protected methods are in place for testing -no guarantees are made.
- * 
- * Inner classes have no synchronization guarantees; they should be 
manipulated 
- * in these classes and not externally.
- * 
- * Note that as well as some methods marked visible for testing, there
- * is the option for the time generator method, {@link #now()} to
- * be overridden so that a repeatable time series can be used.
- * 
- */
-public class RoleHistory {
-  protected static final Logger log =
-LoggerFactory.getLogger(RoleHistory.class);
-  private final List providerRoles;
-  /** the roles in here are shared with App State */
-  private final Map roleStatusMap = new HashMap<>();
-  private final AbstractClusterServices recordFactory;
-
-  private long startTime;
-
-  /** Time when saved */
-  private final Timestamp saveTime = new Timestamp(0);
-
-  /** If the history was loaded, the time at which the history was saved.
-   * That is: the time the data was valid */
-  private final Timestamp thawedDataTime = new Timestamp(0);
-  
-  private NodeMap nodemap;
-  private int roleSize;
-  private final BoolMetric dirty = new BoolMetric(false);
-  private FileSystem filesystem;
-  private Path historyPath;
-  private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
-
-  /**
-   * When were the nodes updated in a {@link #onNodesUpdated(List)} call?
-   * If zero: never.
-   */
-  private final Timestamp nodesUpdatedTime = new Timestamp(0);
-  private final BoolMetric nodeUpdateReceived = new BoolMetric(false);
-
-  private OutstandingRequestTracker 

[43/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
deleted file mode 100644
index 333859e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/slideram-log4j.properties
+++ /dev/null
@@ -1,68 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-# This is the log4j configuration for Slider Application Master
-
-# Log rotation based on size (100KB) with a max of 10 backup files
-log4j.rootLogger=INFO, amlog
-log4j.threshhold=ALL
-log4j.appender.amlog=org.apache.log4j.RollingFileAppender
-log4j.appender.amlog.layout=org.apache.log4j.PatternLayout
-log4j.appender.amlog.File=${LOG_DIR}/slider.log
-log4j.appender.amlog.MaxFileSize=1MB
-log4j.appender.amlog.MaxBackupIndex=10
-
-# log layout skips stack-trace creation operations by avoiding line numbers 
and method
-log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - 
%m%n
-
-# debug edition is much more expensive
-#log4j.appender.amlog.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} 
(%F:%M(%L)) - %m%n
-
-# configure stderr
-# set the conversion pattern of stderr
-# Print the date in ISO 8601 format
-log4j.appender.stderr=org.apache.log4j.ConsoleAppender
-log4j.appender.stderr.Target=System.err
-log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
-log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} - 
%m%n
-
-log4j.appender.subprocess=org.apache.log4j.ConsoleAppender
-log4j.appender.subprocess.layout=org.apache.log4j.PatternLayout
-log4j.appender.subprocess.layout.ConversionPattern=[%c{1}]: %m%n
-#log4j.logger.org.apache.slider.yarn.appmaster.SliderAppMasterer.master=INFO,subprocess
-
-# for debugging Slider
-#log4j.logger.org.apache.slider=DEBUG
-
-# uncomment to debug service lifecycle issues
-#log4j.logger.org.apache.hadoop.yarn.service.launcher=DEBUG
-#log4j.logger.org.apache.hadoop.yarn.service=DEBUG
-
-# uncomment for YARN operations
-#log4j.logger.org.apache.hadoop.yarn.client=DEBUG
-
-# uncomment this to debug security problems
-#log4j.logger.org.apache.hadoop.security=DEBUG
-
-#crank back on some noise
-log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
-log4j.logger.org.apache.hadoop.hdfs=WARN
-log4j.logger.org.apache.hadoop.hdfs.shortcircuit=ERROR
-
-log4j.logger.org.apache.zookeeper=WARN
-log4j.logger.org.apache.curator.framework.state=ERROR
-log4j.logger.org.apache.curator.framework.imps=WARN

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
new file mode 100644
index 000..58c8e27
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf/yarnservice-log4j.properties
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in 

[38/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index 647bfe9..a044838 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -24,30 +24,24 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.AbstractLauncher;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
 import org.apache.hadoop.yarn.service.ServiceContext;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.PublishedConfiguration;
+import org.apache.hadoop.yarn.service.utils.PublishedConfigurationOutputter;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -55,23 +49,18 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.regex.Pattern;
 
-import static org.apache.slider.api.ServiceApiConstants.*;
-import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
+import static 
org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants.*;
 
 /**
  * This is a factoring out of methods handy for providers. It's bonded to a log
  * at construction time.
  */
-public class ProviderUtils implements RoleKeys, SliderKeys {
+public class ProviderUtils implements YarnServiceConstants {
 
   protected static final Logger log =
   LoggerFactory.getLogger(ProviderUtils.class);
@@ -174,46 +163,22 @@ public class ProviderUtils implements RoleKeys, 
SliderKeys {
   }
 
   /**
-   * Get resource requirements from a String value. If value isn't specified,
-   * use the default value. If value is greater than max, use the max value.
-   * @param val string value
-   * @param defVal default value
-   * @param maxVal maximum value

[03/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
deleted file mode 100644
index 9c5708f..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.java
+++ /dev/null
@@ -1,188 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.junit.Assert;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.junit.Assert.assertNotNull;
-
-/**
- * This is an evolving engine to mock YARN operations.
- */
-public class MockYarnEngine {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(MockYarnEngine.class);
-
-  private MockYarnCluster cluster;
-  private Allocator allocator;
-  private List pending = new ArrayList<>();
-
-  private ApplicationId appId = new MockApplicationId(0, 0);
-
-  private ApplicationAttemptId attemptId = new MockApplicationAttemptId(appId,
-  1);
-
-  @Override
-  public String toString() {
-return "MockYarnEngine " + cluster + " + pending=" + pending.size();
-  }
-
-  public int containerCount() {
-return cluster.containersInUse();
-  }
-
-  public MockYarnEngine(int clusterSize, int containersPerNode) {
-cluster = new MockYarnCluster(clusterSize, containersPerNode);
-allocator = new Allocator(cluster);
-  }
-
-  public MockYarnCluster getCluster() {
-return cluster;
-  }
-
-  public Allocator getAllocator() {
-return allocator;
-  }
-
-  /**
-   * Allocate a container from a request. The containerID will be
-   * unique, nodeId and other fields chosen internally with
-   * no such guarantees; resource and priority copied over
-   * @param request request
-   * @return container
-   */
-  public Container allocateContainer(AMRMClient.ContainerRequest request) {
-MockContainer allocated = allocator.allocate(request);
-if (allocated != null) {
-  MockContainerId id = (MockContainerId)allocated.getId();
-  id.setApplicationAttemptId(attemptId);
-}
-return allocated;
-  }
-
-  MockYarnCluster.MockYarnClusterContainer releaseContainer(ContainerId
-  containerId) {
-return cluster.release(containerId);
-  }
-
-  /**
-   * Process a list of operations -release containers to be released,
-   * allocate those for which there is space (but don't rescan the list after
-   * the scan).
-   * @param ops
-   * @return
-   */
-  public List execute(List ops) {
-return execute(ops, new ArrayList<>());
-  }
-
-  /**
-   * Process a list of operations -release containers to be released,
-   * allocate those for which there is space (but don't rescan the list after
-   * the scan). Unsatisifed entries 

[54/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
deleted file mode 100644
index 7baa284..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/CommandLineBuilder.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.containerlaunch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Build a single command line to include in the container commands;
- * Special support for JVM command buildup.
- */
-public class CommandLineBuilder {
-  protected final List argumentList = new ArrayList<>(20);
-
-  /**
-   * Add an entry to the command list
-   * @param args arguments -these will be converted strings
-   */
-  public void add(Object... args) {
-for (Object arg : args) {
-  argumentList.add(arg.toString());
-}
-  }
-
-  // Get the number of arguments
-  public int size() {
-return argumentList.size();
-  }
-  
-  /**
-   * Append the output and error files to the tail of the command
-   * @param stdout out
-   * @param stderr error. Set this to null to append into stdout
-   */
-  public void addOutAndErrFiles(String stdout, String stderr) {
-Preconditions.checkNotNull(stdout, "Null output file");
-Preconditions.checkState(!stdout.isEmpty(), "output filename invalid");
-// write out the path output
-argumentList.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" +
- stdout);
-if (stderr != null) {
-  argumentList.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" 
+
-   stderr);
-} else {
-  argumentList.add("2>&1");
-}
-  }
-
-  /**
-   * This just returns the command line
-   * @see #build()
-   * @return the command line
-   */
-  @Override
-  public String toString() {
-return build();
-  }
-
-  /**
-   * Build the command line
-   * @return the command line
-   */
-  public String build() {
-return SliderUtils.join(argumentList, " ");
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
deleted file mode 100644
index fcbb69b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you 

[63/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
new file mode 100644
index 000..7b22e3e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/JsonSerDeser.java
@@ -0,0 +1,249 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.map.DeserializationConfig;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.codehaus.jackson.map.SerializationConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ * This class is NOT thread safe; it constructs an object mapper
+ * as an instance field.
+ * @param 
+ */
+public class JsonSerDeser {
+
+  private static final Logger log = 
LoggerFactory.getLogger(JsonSerDeser.class);
+  private static final String UTF_8 = "UTF-8";
+
+  private final Class classType;
+  private final ObjectMapper mapper;
+
+  /**
+   * Create an instance bound to a specific type
+   * @param classType class type
+   */
+  public JsonSerDeser(Class classType) {
+this.classType = classType;
+this.mapper = new ObjectMapper();
+mapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, 
false);
+  }
+
+  public JsonSerDeser(Class classType, PropertyNamingStrategy 
namingStrategy) {
+this(classType);
+mapper.setPropertyNamingStrategy(namingStrategy);
+  }
+
+  /**
+   * Convert from JSON
+   * @param json input
+   * @return the parsed JSON
+   * @throws IOException IO
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public T fromJson(String json)
+throws IOException, JsonParseException, JsonMappingException {
+try {
+  return mapper.readValue(json, classType);
+} catch (IOException e) {
+  log.error("Exception while parsing json : " + e + "\n" + json, e);
+  throw e;
+}
+  }
+
+  /**
+   * Convert from a JSON file
+   * @param jsonFile input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+  public T fromFile(File jsonFile)
+throws IOException, JsonParseException, JsonMappingException {
+File absoluteFile = jsonFile.getAbsoluteFile();
+try {
+  return mapper.readValue(absoluteFile, classType);
+} catch (IOException e) {
+  log.error("Exception while parsing json file {}", absoluteFile, e);
+  throw e;
+}
+  }
+
+  /**
+   * Convert from a JSON file
+   * @param resource input file
+   * @return the parsed JSON
+   * @throws IOException IO problems
+   * @throws JsonMappingException failure to map from the JSON to this class
+   */
+ public T fromResource(String resource)
+throws IOException, JsonParseException, JsonMappingException {
+

[75/75] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi

[jianhe]

YARN-7113. Clean up packaging and dependencies for yarn-native-services. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef230a3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef230a3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef230a3a

Branch: refs/heads/yarn-native-services
Commit: ef230a3a27dafa04bb2d379168dad0de478ab9c0
Parents: 5dc3432
Author: Jian He 
Authored: Tue Aug 29 11:09:00 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:45:12 2017 -0700

--
 NOTICE.txt  |   15 +
 .../resources/assemblies/hadoop-yarn-dist.xml   |8 -
 .../assemblies/hadoop-yarn-services-api.xml |   36 -
 .../assemblies/hadoop-yarn-services-dist.xml|   30 -
 hadoop-project/pom.xml  |   19 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   37 +-
 .../hadoop-yarn-services-api/pom.xml|  104 +-
 .../yarn/service/webapp/ApiServerWebApp.java|4 +-
 .../src/main/resources/webapps/api-server/app   |   16 +
 .../resources/webapps/services-rest-api/app |   16 -
 .../hadoop-yarn-services-core/pom.xml   |  213 +---
 .../service/client/params/ActionKDiagArgs.java  |   76 --
 .../yarn/service/client/params/ClientArgs.java  |5 -
 .../registry/YarnRegistryViewForProviders.java  |8 +-
 .../yarn/service/utils/KerberosDiags.java   |  680 ---
 .../hadoop/yarn/service/utils/SliderUtils.java  | 1088 --
 .../hadoop/yarn/service/ServiceTestUtils.java   |   28 +
 .../hadoop/yarn/service/TestServiceApiUtil.java |   38 +-
 .../yarn/service/TestYarnNativeServices.java|   10 +-
 .../yarn/service/client/TestServiceCLI.java |1 -
 .../yarn/service/conf/TestAppJsonResolve.java   |   30 +-
 .../service/conf/TestLoadExampleAppJson.java|   11 +-
 .../providers/TestAbstractClientProvider.java   |   10 +-
 hadoop-yarn-project/pom.xml |4 +
 24 files changed, 176 insertions(+), 2311 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef230a3a/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index c41972b..e40f3bf 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -576,3 +576,18 @@ The binary distribution of this product bundles binaries of
 software.amazon.ion:ion-java 1.0.1,
 which has the following notices:
  * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All 
Rights Reserved.
+
+The binary distribution of this product bundles binaries of
+JCommander (https://github.com/cbeust/jcommander),
+which has the following notices:
+ * Copyright 2010 Cedric Beust ced...@beust.com
+
+The binary distribution of this product bundles binaries of
+snakeyaml (https://bitbucket.org/asomov/snakeyaml),
+which has the following notices:
+ * Copyright (c) 2008, http://www.snakeyaml.org
+
+The binary distribution of this product bundles binaries of
+swagger-annotations (https://github.com/swagger-api/swagger-core),
+which has the following notices:
+ * Copyright 2016 SmartBear Software

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef230a3a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8aeeabd..8b3d292 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,10 +98,6 @@
   etc/hadoop
 
 
-  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}
-  
/share/hadoop/${hadoop.component}/lib/services
-
-
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
   
/share/hadoop/${hadoop.component}/sources
   
@@ -109,10 +105,6 @@
   
 
 
-  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}
-  
/share/hadoop/${hadoop.component}/lib/services-api
-
-
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef230a3a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml 

[06/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
deleted file mode 100644
index 8686479..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
+++ /dev/null
@@ -1,430 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import static 
org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority;
-import static 
org.apache.slider.server.appmaster.state.ContainerPriority.extractRole;
-
-/**
- * Test app state RM operations.
- */
-public class TestMockAppStateRMOperations extends BaseMockAppStateTest
-implements MockRoles {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(BaseMockAppStateTest.class);
-
-  @Override
-  public String getTestName() {
-return "TestMockAppStateRMOperations";
-  }
-
-  //@Test
-  public void testPriorityOnly() throws Throwable {
-assertEquals(5, extractRole(buildPriority(5, false)));
-  }
-
-  //@Test
-  public void testPriorityRoundTrip() throws Throwable {
-assertEquals(5, extractRole(buildPriority(5, false)));
-  }
-
-  //@Test
-  public void testPriorityRoundTripWithRequest() throws Throwable {
-int priority = buildPriority(5, false);
-assertEquals(5, extractRole(priority));
-  }
-
-  //@Test
-  public void testMockAddOp() throws Throwable {
-getRole0Status().setDesired(1);
-List ops = appState.reviewRequestAndReleaseNodes();
-assertListLength(ops, 1);
-ContainerRequestOperation operation = 
(ContainerRequestOperation)ops.get(0);
-int priority = operation.getRequest().getPriority().getPriority();
-assertEquals(extractRole(priority), getRole0Status().getKey());
-MockRMOperationHandler handler = new MockRMOperationHandler();
-handler.execute(ops);
-
-AbstractRMOperation op = handler.getFirstOp();
-assertTrue(op instanceof ContainerRequestOperation);
-  }
-
-  /**
-   * Test of a flex up and down op which verifies that outstanding
-   * requests are cancelled first.
-   * 
-   *   request 5 nodes, assert 5 request made
-   *   allocate 1 of them
-   *   flex cluster size to 3
-   *   assert this generates 2 cancel requests
-   * 
-   */
-  //@Test
-  public void testRequestThenCancelOps() 

[69/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
new file mode 100644
index 000..c3a2752
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -0,0 +1,892 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
+import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
+import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import 

[67/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
new file mode 100644
index 000..cb7131e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -0,0 +1,494 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.ContainerFailureTracker;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import org.apache.hadoop.yarn.service.ServiceMetrics;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.SingleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
+import org.apache.hadoop.yarn.service.monitor.probe.Probe;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import static org.apache.hadoop.yarn.api.records.ContainerExitStatus.*;
+import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.START;
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
+import static org.apache.hadoop.yarn.service.component.ComponentState.*;
+import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
+
+public class Component implements EventHandler {
+  private static final Logger LOG = LoggerFactory.getLogger(Component.class);
+
+  private org.apache.hadoop.yarn.service.api.records.Component componentSpec;
+  private long allocateId;
+  private Priority priority;
+  private ServiceMetrics componentMetrics;
+  private ServiceScheduler scheduler;
+  private ServiceContext context;
+  private AMRMClientAsync amrmClient;
+  private AtomicLong instanceIdCounter = new AtomicLong();
+  private Map compInstances =
+  new ConcurrentHashMap<>();
+  // component instances to be assigned with a container
+  private List pendingInstances = new LinkedList<>();
+  private ContainerFailureTracker 

[59/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
deleted file mode 100644
index 8c968dc..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service;
-
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.UpdatedContainer;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.api.constants.ServiceApiConstants;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.ConfigFile;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEvent;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.service.component.ComponentEvent;
-import org.apache.hadoop.yarn.service.component.ComponentEventType;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.hadoop.yarn.service.provider.ProviderUtils;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import 

[64/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
new file mode 100644
index 000..add2475
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/registry/YarnRegistryViewForProviders.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.registry;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.PathNotFoundException;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
+
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.join;
+
+/**
+ * Registry view for providers. This tracks where the service
+ * is registered, offers access to the record and other things.
+ */
+public class YarnRegistryViewForProviders {
+  private static final Log LOG =
+  LogFactory.getLog(YarnRegistryViewForProviders.class);
+
+  private final RegistryOperations registryOperations;
+  private final String user;
+  private final String sliderServiceClass;
+  private final String instanceName;
+  /**
+   * Record used where the service registered itself.
+   * Null until the service is registered
+   */
+  private ServiceRecord selfRegistration;
+
+  /**
+   * Path where record was registered
+   * Null until the service is registered
+   */
+  private String selfRegistrationPath;
+
+  public YarnRegistryViewForProviders(RegistryOperations registryOperations,
+  String user,
+  String sliderServiceClass,
+  String instanceName,
+  ApplicationAttemptId applicationAttemptId) {
+Preconditions.checkArgument(registryOperations != null,
+"null registry operations");
+Preconditions.checkArgument(user != null, "null user");
+Preconditions.checkArgument(SliderUtils.isSet(sliderServiceClass),
+"unset service class");
+Preconditions.checkArgument(SliderUtils.isSet(instanceName),
+"instanceName");
+Preconditions.checkArgument(applicationAttemptId != null,
+"null applicationAttemptId");
+this.registryOperations = registryOperations;
+this.user = user;
+this.sliderServiceClass = sliderServiceClass;
+this.instanceName = instanceName;
+  }
+
+  public String getUser() {
+return user;
+  }
+
+
+  private void setSelfRegistration(ServiceRecord selfRegistration) {
+this.selfRegistration = selfRegistration;
+  }
+
+  /**
+   * Get the path to where the service has registered itself.
+   * Null until the service is registered
+   * @return the service registration path.
+   */
+  public String getSelfRegistrationPath() {
+return selfRegistrationPath;
+  }
+
+  /**
+   * Get the absolute path to where the service has registered itself.
+   * This includes the base registry path
+   * Null until the service is registered
+   * @return the service 

[56/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
deleted file mode 100644
index c2866cf..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client.params;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.UsageException;
-import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
-
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
-import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
-import java.io.File;
-
-/**
- * Registry actions
- * 
- * --instance {app name}, if  a / is in it, refers underneath?
- * --dest {destfile}
- * --list : list instances of slider service
- * --listfiles 
- */
-@Parameters(commandNames = {ACTION_REGISTRY},
-commandDescription = DESCRIBE_ACTION_REGISTRY)
-
-public class ActionRegistryArgs extends AbstractActionArgs {
-
-  public static final String USAGE =
-  "Usage: " + SliderActions.ACTION_REGISTRY
-  + " ("
-  + Arguments.ARG_LIST + "|"
-  + Arguments.ARG_LISTCONF + "|"
-  + Arguments.ARG_LISTEXP + "|"
-  + Arguments.ARG_LISTFILES + "|"
-  + Arguments.ARG_GETCONF + "|"
-  + Arguments.ARG_GETEXP + "> "
-  + Arguments.ARG_NAME + "  "
-  + " )"
-  + "[" + Arguments.ARG_VERBOSE + "] "
-  + "[" + Arguments.ARG_USER + "] "
-  + "[" + Arguments.ARG_OUTPUT + "  ] "
-  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
-  + "[" + Arguments.ARG_FORMAT + " ] "
-  + System.getProperty("line.separator")
-  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
-  ;
-  public ActionRegistryArgs() {
-  }
-
-  public ActionRegistryArgs(String name) {
-this.name = name;
-  }
-
-  @Override
-  public String getActionName() {
-return ACTION_REGISTRY;
-  }
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  @Override
-  public int getMinParams() {
-return 0;
-  }
-  
-  @Parameter(names = {ARG_LIST}, 
-  description = "list services")
-  public boolean list;
-
-  @Parameter(names = {ARG_LISTCONF}, 
-  description = "list configurations")
-  public boolean listConf;
-
-  @Parameter(names = {ARG_GETCONF},
-  description = "get configuration")
-  public String getConf;
-
-  @Parameter(names = {ARG_LISTEXP},
- description = "list exports")
-  public boolean listExports;
-
-  @Parameter(names = {ARG_GETEXP},
- description = "get export")
-  public String getExport;
-
-  @Parameter(names = {ARG_LISTFILES},
-  description = "list files")
-  public String listFiles;
-
-  @Parameter(names = {ARG_GETFILES},
-  description = "get files")
-  public String getFiles;
-
-  //--format 
-  @Parameter(names = ARG_FORMAT,
-  description = "Format for a response: ")
-  public String format = ConfigFormat.XML.toString() ;
-
-  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
-  description = "Output destination")
-  public File out;
-
-  @Parameter(names 

[30/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
deleted file mode 100644
index f1bf2ad..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.ActionAMSuicideArgs;
-import org.apache.slider.common.params.ActionClientArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.slider.common.params.ActionDiagnosticArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.slider.common.params.ActionFreezeArgs;
-import org.apache.slider.common.params.ActionKeytabArgs;
-import org.apache.slider.common.params.ActionNodesArgs;
-import org.apache.slider.common.params.ActionKillContainerArgs;
-import org.apache.slider.common.params.ActionListArgs;
-import org.apache.slider.common.params.ActionRegistryArgs;
-import org.apache.slider.common.params.ActionResolveArgs;
-import org.apache.slider.common.params.ActionResourceArgs;
-import org.apache.slider.common.params.ActionStatusArgs;
-import org.apache.slider.common.params.ActionThawArgs;
-import org.apache.slider.common.params.ActionUpgradeArgs;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.SliderException;
-
-import java.io.IOException;
-
-/**
- * Interface of those method calls in the slider API that are intended
- * for direct public invocation.
- * 
- * Stability: evolving
- */
-public interface SliderClientAPI extends Service {
-
-  int actionDestroy(String clustername) throws YarnException, IOException;
-
-  /**
-   * AM to commit an asynchronous suicide
-   */
-  int actionAmSuicide(String clustername,
-  ActionAMSuicideArgs args) throws YarnException, IOException;
-
-  /**
-   * Manage keytabs leveraged by slider
-   *
-   * @param keytabInfo the arguments needed to manage the keytab
-   * @throws YarnException Yarn problems
-   * @throws IOException other problems
-   * @throws BadCommandArgumentsException bad arguments.
-   */
-  int actionKeytab(ActionKeytabArgs keytabInfo)
-  throws YarnException, IOException;
-
-  /**
-   * Manage file resources leveraged by slider
-   *
-   * @param resourceInfo the arguments needed to manage the resource
-   * @throws YarnException Yarn problems
-   * @throws IOException other problems
-   * @throws BadCommandArgumentsException bad arguments.
-   */
-  int actionResource(ActionResourceArgs resourceInfo)
-  throws YarnException, IOException;
-
-  /**
-   * Perform client operations such as install or configure
-   *
-   * @param clientInfo the arguments needed for client operations
-   *
-   * @throws SliderException bad arguments.
-   * @throws IOException problems related to package and destination folders
-   */
-  int actionClient(ActionClientArgs clientInfo)
-  throws IOException, YarnException;
-
-  /**
-   * Update the cluster specification
-   *
-   * @param clustername cluster name
-   * @param buildInfo the arguments needed to update the cluster
-   * @throws YarnException Yarn problems
-   * @throws 

[32/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
deleted file mode 100644
index b28257f..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/ClientUtils.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.NotFoundException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.retrieve.RegistryRetriever;
-
-import java.io.File;
-import java.io.IOException;
-
-import static 
org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
-import static 
org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-
-public class ClientUtils {
-  public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
-  String user, String name) throws IOException, SliderException {
-return lookupServiceRecord(rops, user, null, name);
-  }
-
-  public static ServiceRecord lookupServiceRecord(RegistryOperations rops,
-  String user, String type, String name) throws IOException,
-  SliderException {
-if (StringUtils.isEmpty(user)) {
-  user = currentUser();
-} else {
-  user = RegistryPathUtils.encodeForRegistry(user);
-}
-if (StringUtils.isEmpty(type)) {
-  type = SliderKeys.APP_TYPE;
-}
-
-String path = servicePath(user, type, name);
-return resolve(rops, path);
-  }
-
-  public static ServiceRecord resolve(RegistryOperations rops, String path)
-  throws IOException, SliderException {
-try {
-  return rops.resolve(path);
-} catch (PathNotFoundException | NoRecordException e) {
-  throw new NotFoundException(e.getPath().toString(), e);
-}
-  }
-
-  public static PublishedConfiguration getConfigFromRegistry(
-  RegistryOperations rops, Configuration configuration,
-  String configName, String appName, String user, boolean external)
-  throws IOException, SliderException {
-ServiceRecord instance = lookupServiceRecord(rops, user, appName);
-
-RegistryRetriever retriever = new RegistryRetriever(configuration, 
instance);
-PublishedConfigSet configurations = retriever.getConfigurations(external);
-
-PublishedConfiguration published = retriever.retrieveConfiguration(
-configurations, configName, external);
-return published;
-  }
-
-  public static String saveOrReturnConfig(PublishedConfiguration published,
-  String format, File destPath, String fileName)
-  throws BadCommandArgumentsException, IOException {
-ConfigFormat configFormat = ConfigFormat.resolve(format);
-if (configFormat == null) {
-  throw new BadCommandArgumentsException(
-  "Unknown/Unsupported format %s 

[55/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
deleted file mode 100644
index 982a114..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/compinstance/ComponentInstance.java
+++ /dev/null
@@ -1,493 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.compinstance;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.client.api.NMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.service.ServiceScheduler;
-import org.apache.hadoop.yarn.service.api.records.ContainerState;
-import org.apache.hadoop.yarn.service.component.Component;
-import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
-import org.apache.hadoop.yarn.state.SingleArcTransition;
-import org.apache.hadoop.yarn.state.StateMachine;
-import org.apache.hadoop.yarn.state.StateMachineFactory;
-import org.apache.hadoop.yarn.util.BoundedAppender;
-import org.apache.hadoop.yarn.service.utils.SliderUtils;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.servicemonitor.probe.ProbeStatus;
-import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.Date;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
-import static 
org.apache.hadoop.yarn.api.records.ContainerExitStatus.KILLED_BY_APPMASTER;
-import static org.apache.hadoop.yarn.api.records.ContainerState.COMPLETE;
-import static 
org.apache.hadoop.yarn.service.compinstance.ComponentInstanceEventType.*;
-import static 
org.apache.hadoop.yarn.service.compinstance.ComponentInstanceState.*;
-
-public class ComponentInstance implements EventHandler,
-Comparable {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(ComponentInstance.class);
-
-  private  StateMachine stateMachine;
-  private Component component;
-  private final ReadLock readLock;
-  private final WriteLock writeLock;
-
-  private ComponentInstanceId compInstanceId = null;
-  private Path compInstanceDir;
-  private Container container;
-  private YarnRegistryViewForProviders yarnRegistryOperations;
-  private FileSystem fs;
-  private boolean timelineServiceEnabled = false;
-  private 

[50/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
deleted file mode 100644
index 405f690..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- * 
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
-  public String applicationId;
-  public String applicationAttemptId;
-  public String name;
-  public String applicationType;
-  public String user;
-  public String queue;
-  public String host;
-  public Integer rpcPort;
-  public String state;
-  public String diagnostics;
-  public String url;
-  /**
-   * This value is non-null only when a report is generated from a submission 
context.
-   * The YARN {@link ApplicationReport} structure does not propagate this value
-   * from the RM.
-   */
-  public Long submitTime;
-  public Long startTime;
-  public Long finishTime;
-  public String finalStatus;
-  public String origTrackingUrl;
-  public Float progress;
-  
-  public SerializedApplicationReport() {
-  }
-  
-  public SerializedApplicationReport(ApplicationReport report) {
-this.applicationId = report.getApplicationId().toString();
-ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
-this.applicationAttemptId = attemptId != null ? attemptId.toString() : 
"N/A";
-this.name = report.getName();
-this.applicationType = report.getApplicationType();
-this.user = report.getUser();
-this.queue = report.getQueue();
-this.host = report.getHost();
-this.rpcPort = report.getRpcPort();
-this.state = report.getYarnApplicationState().toString();
-this.diagnostics = report.getDiagnostics();
-this.startTime = report.getStartTime();
-this.finishTime = report.getFinishTime();
-FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
-this.finalStatus = appStatus == null ? "" : appStatus.toString();
-this.progress = report.getProgress();
-this.url = report.getTrackingUrl();
-this.origTrackingUrl= report.getOriginalTrackingUrl();
-  }
-
-  @Override
-  public String toString() {
-try {
-  return ApplicationReportSerDeser.toString(this);
-} catch (IOException e) {
-  return super.toString();
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
--
diff --git 

[60/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
new file mode 100644
index 000..b742553
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/timelineservice/TestServiceTimelinePublisher.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.timelineservice;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.impl.TimelineV2ClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
+import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test class for ServiceTimelinePublisher.
+ */
+public class TestServiceTimelinePublisher {
+  private TimelineV2Client timelineClient;
+  private Configuration config;
+  private ServiceTimelinePublisher serviceTimelinePublisher;
+  private static String SERVICE_NAME = "HBASE";
+  private static String SERVICEID = "application_1490093646524_0005";
+  private static String ARTIFACTID = "ARTIFACTID";
+  private static String COMPONENT_NAME = "DEFAULT";
+  private static String CONTAINER_ID =
+  "container_e02_1490093646524_0005_01_01";
+  private static String CONTAINER_IP =
+  "localhost";
+  private static String CONTAINER_HOSTNAME =
+  "cnl124-localhost.site";
+  private static String CONTAINER_BAREHOST =
+  "localhost.com";
+
+  @Before
+  public void setUp() throws Exception {
+config = new Configuration();
+config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+timelineClient =
+new DummyTimelineClient(ApplicationId.fromString(SERVICEID));
+serviceTimelinePublisher = new ServiceTimelinePublisher(timelineClient);
+timelineClient.init(config);
+serviceTimelinePublisher.init(config);
+timelineClient.start();
+serviceTimelinePublisher.start();
+  }
+
+  @After
+  public void tearDown() throws Exception {
+if 

[74/75] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef230a3a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
index 6e6f4dd..7e53d18 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -24,35 +24,17 @@ import 
org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
 import org.apache.hadoop.yarn.service.client.params.SliderActions;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
 import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
-import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
 import org.apache.hadoop.yarn.service.exceptions.SliderException;
-import org.apache.zookeeper.server.util.KerberosUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,32 +45,19 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
-import java.io.Serializable;
-import java.net.InetSocketAddress;
 import java.net.ServerSocket;
-import java.net.Socket;
 import java.net.URL;
 import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
 import java.util.Enumeration;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPOutputStream;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipOutputStream;
 
 /**
  * These are slider-specific Util methods
@@ -97,41 +66,6 @@ public final class SliderUtils {
 
   private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
 
-  /**
-   * Atomic bool to track whether or not process security has already been
-   * turned on (prevents re-entrancy)
-   */
-  private static final AtomicBoolean processSecurityAlreadyInitialized =
-  new AtomicBoolean(false);
-  public static final String JAVA_SECURITY_KRB5_REALM =
-  "java.security.krb5.realm";
-  public static final String JAVA_SECURITY_KRB5_KDC = "java.security.krb5.kdc";
-
-  /**
-   * Winutils
-   */
-  public static final String WINUTILS = "WINUTILS.EXE";
-  /**
-   * name of openssl program
-   */
-  public static final String OPENSSL = "openssl";
-
-  /**
-   * name of python program
-   */
-  public static final String PYTHON = "python";
-
-  /**
-   * type of docker standalone service
-   */
-  public static final String DOCKER = "docker";
-  /**
-   * type of docker on yarn service
-   */
-  public static final String DOCKER_YARN = "yarn_docker";
-

[17/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
deleted file mode 100644
index ba923bc..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ /dev/null
@@ -1,2120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ServiceApiConstants;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.proto.Messages.ComponentCountProto;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.ApplicationState;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.types.ApplicationLivenessInformation;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.RoleStatistics;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
-import org.apache.slider.server.appmaster.management.MetricsConstants;
-import org.apache.hadoop.yarn.service.metrics.ServiceMetrics;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.operations.UpdateBlacklistOperation;
-import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;

[23/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
deleted file mode 100644
index bf71861..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/restclient/UgiJerseyBinding.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.restclient;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * 
- *   Usage: create an instance, then when creating a Jersey Client
- *   pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
- */
-public class UgiJerseyBinding implements
-HttpURLConnectionFactory {
-  private static final Logger log =
-  LoggerFactory.getLogger(UgiJerseyBinding.class);
-
-  private final UrlConnectionOperations operations;
-  private final URLConnectionClientHandler handler;
-
-  /**
-   * Construct an instance
-   * @param operations operations instance
-   */
-  @SuppressWarnings("ThisEscapedInObjectConstruction")
-  public UgiJerseyBinding(UrlConnectionOperations operations) {
-Preconditions.checkArgument(operations != null, "Null operations");
-this.operations = operations;
-handler = new URLConnectionClientHandler(this);
-  }
-
-  /**
-   * Create an instance off the configuration. The SPNEGO policy
-   * is derived from the current UGI settings.
-   * @param conf config
-   */
-  public UgiJerseyBinding(Configuration conf) {
-this(new UrlConnectionOperations(conf));
-  }
-
-  /**
-   * Get a URL connection. 
-   * @param url URL to connect to
-   * @return the connection
-   * @throws IOException any problem. {@link AuthenticationException} 
-   * errors are wrapped
-   */
-  @Override
-  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
-try {
-  // open a connection handling status codes and so redirections
-  // but as it opens a connection, it's less useful than you think.
-
-  return operations.openConnection(url);
-} catch (AuthenticationException e) {
-  throw new IOException(e);
-}
-  }
-
-  public UrlConnectionOperations getOperations() {
-return operations;
-  }
-
-  public URLConnectionClientHandler getHandler() {
-return handler;
-  }
-  
-  /**
-   * Get the SPNEGO flag (as found in the operations instance
-   * @return the spnego policy
-   */
-  public boolean isUseSpnego() {
-return operations.isUseSpnego();
-  }
-
-
-  /**
-   * Uprate error codes 400 and up into faults; 
-   * 
-   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
-   */
-  public static 

[47/75] [abbrv] hadoop git commit: Rebased on to trunk, fix conflicts

[jianhe]

Rebased on to trunk, fix conflicts


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5dc3432b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5dc3432b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5dc3432b

Branch: refs/heads/yarn-native-services
Commit: 5dc3432bd41fd66aa233012a19d2326fe86b5ffe
Parents: db5888e
Author: Jian He 
Authored: Mon Aug 28 15:05:01 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:45:12 2017 -0700

--
 .../main/java/org/apache/hadoop/yarn/util/BoundedAppender.java   | 4 
 .../server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java   | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc3432b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
index 1a1593a..e2cb2ee 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BoundedAppender.java
@@ -115,6 +115,10 @@ public class BoundedAppender {
 return messages.length();
   }
 
+  public int getLimit() {
+return limit;
+  }
+
   /**
* Get a string representation of the actual contents, displaying also a
* header and ellipses when there was a truncate.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dc3432b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 44d6986..c2750c9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -1327,7 +1327,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, 
Recoverable {
 // AFTER the initial saving on app-attempt-start
 // These fields can be visible from outside only after they are saved in
 // StateStore
-BoundedAppender diags = new BoundedAppender(diagnostics.limit);
+BoundedAppender diags = new BoundedAppender(diagnostics.getLimit());
 
 // don't leave the tracking URL pointing to a non-existent AM
 if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[70/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 000..c64b1b5
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+  private Integer code = null;
+  private String message = null;
+  private String fields = null;
+
+  /**
+   **/
+  public Error code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  /**
+   **/
+  public Error message(String message) {
+this.message = message;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("message")
+  public String getMessage() {
+return message;
+  }
+
+  public void setMessage(String message) {
+this.message = message;
+  }
+
+  /**
+   **/
+  public Error fields(String fields) {
+this.fields = fields;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("fields")
+  public String getFields() {
+return fields;
+  }
+
+  public void setFields(String fields) {
+this.fields = fields;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+Error error = (Error) o;
+return Objects.equals(this.code, error.code)
+&& Objects.equals(this.message, error.message)
+&& Objects.equals(this.fields, error.fields);
+  }
+
+  @Override
+  public int hashCode() {
+return Objects.hash(code, message, fields);
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder();
+sb.append("class Error {\n");
+
+sb.append("code: ").append(toIndentedString(code)).append("\n");
+sb.append("message: ").append(toIndentedString(message)).append("\n");
+sb.append("fields: ").append(toIndentedString(fields)).append("\n");
+sb.append("}");
+return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+if (o == null) {
+  return "null";
+}
+return o.toString().replace("\n", "\n");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
--
diff --git 

[71/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
new file mode 100644
index 000..fb2fd16
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -0,0 +1,654 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import 

[27/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
deleted file mode 100644
index fc57c82..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ /dev/null
@@ -1,2548 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.Slider;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.apache.zookeeper.server.util.KerberosUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.Serializable;
-import java.io.StringWriter;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.EnumSet;
-import 

[35/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 000..415392a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1699 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+  private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+  /**
+   * Atomic 

[13/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
deleted file mode 100644
index 3e9b764..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.core.registry.docstore.UriMap;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_REGEXP;
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP;
-
-/**
- * This publishes configuration sets
- */
-public class PublisherResource extends AbstractSliderResource {
-  protected static final Logger log =
-  LoggerFactory.getLogger(PublisherResource.class);
-  public static final String EXPORTS_NAME = "exports";
-  public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
-  public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + 
"/{exportname}" ;
-  public static final String SET_NAME =
-  "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
-  public static final String SETNAME = "setname";
-  public static final String CLASSPATH = "/classpath";
-  public static final String CONFIG = "config";
-  
-  public static final String SETNAME_PATTERN = 
-  "{"+ SETNAME+": " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
-  private static final String CONFIG_PATTERN =
-  SETNAME_PATTERN + "/{"+ CONFIG +": " + PUBLISHED_CONFIGURATION_REGEXP + 
"}";
-  private final StateAccessForProviders appState;
-
-  public PublisherResource(WebAppApi slider) {
-super(slider);
-appState = slider.getAppState();
-  }
-
-  private void init(HttpServletResponse res, UriInfo uriInfo) {
-res.setContentType(null);
-log.debug(uriInfo.getRequestUri().toString());
-  }
- 
-  /**
-   * Get a named config set 
-   * @param setname name of the config set
-   * @return the config set
-   * @throws NotFoundException if there was no matching set
-   */
-  private PublishedConfigSet getConfigSet(String setname) {
-PublishedConfigSet configSet =
-appState.getPublishedConfigSet(setname);
-if (configSet == null) {
-  

[57/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
deleted file mode 100644
index 1049698..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ /dev/null
@@ -1,872 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.client;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.CuratorFrameworkFactory;
-import org.apache.curator.retry.RetryNTimes;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
-import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
-import org.apache.hadoop.yarn.service.ClientAMProtocol;
-import org.apache.hadoop.yarn.service.ServiceMaster;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.api.records.Component;
-import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
-import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.ClientArgs;
-import org.apache.hadoop.yarn.service.client.params.CommonArgs;
-import org.apache.hadoop.yarn.service.conf.SliderExitCodes;

[45/75] [abbrv] hadoop git commit: YARN-5244. Documentation required for DNS Server implementation. Contributed by Jon Maron

[jianhe]

YARN-5244. Documentation required for DNS Server implementation. Contributed by 
Jon Maron


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/963f2d94
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/963f2d94
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/963f2d94

Branch: refs/heads/yarn-native-services
Commit: 963f2d94df45902d98df9a1466818ba8f6bfa238
Parents: 394183c
Author: Gour Saha 
Authored: Thu Aug 24 19:05:13 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:45:11 2017 -0700

--
 hadoop-project/src/site/site.xml|   7 +-
 .../native-services/NativeServicesDiscovery.md  | 127 +++
 .../native-services/NativeServicesIntro.md  |   0
 .../src/site/resources/images/dns_overview.png  | Bin 0 -> 41908 bytes
 .../resources/images/dns_record_creation.jpeg   | Bin 0 -> 51911 bytes
 .../resources/images/dns_record_removal.jpeg| Bin 0 -> 58041 bytes
 6 files changed, 133 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/963f2d94/hadoop-project/src/site/site.xml
--
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index e7af227..837ac3f 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -148,7 +148,12 @@
   
   
 
-
+
+
+  
+  
+
+
 
   
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/963f2d94/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
new file mode 100644
index 000..4a048af
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/native-services/NativeServicesDiscovery.md
@@ -0,0 +1,127 @@
+# YARN DNS Server
+## Introduction
+
+The YARN DNS Server provides a standard DNS interface to the information 
posted into the YARN Registry by deployed applications. The DNS service serves 
the following functions:
+
+1. **Exposing existing service­ discovery information via DNS**​­ - 
Information provided in
+the current YARN service registry’s records will be converted into DNS 
entries, thus
+allowing users to discover information about YARN applications using standard 
DNS
+client mechanisms (for e.g. a DNS SRV Record specifying the hostname and port
+number for services).
+2. **Enabling Container to IP mappings​­** - Enables discovery of the IPs 
of containers via
+standard DNS lookups. Given the availability of the records via DNS, container
+name­based communication will be facilitated (e.g. ‘curl
+http://myContainer.myDomain.com/endpoint’).
+
+## Service Properties
+
+The existing YARN Service Registry is leveraged as the source of information 
for the DNS Service.
+
+The following core functions are supported by the DNS­ Server:
+
+###Functional properties
+
+1. Supports creation of DNS records for end­points of the deployed YARN 
applications
+2. Record names remain unchanged during restart of containers and/or 
applications
+3. Supports reverse lookups (name based on IP).
+4. Supports security using the standards defined by The Domain Name System 
Security
+Extensions (DNSSEC)
+5. Highly available
+6. Scalable ­- The service provides the responsiveness (e.g. low­ latency) 
required to
+respond to DNS queries (timeouts yield attempts to invoke other configured name
+servers).
+
+###Deployment properties
+
+1. Supports integration with existing DNS assets (e.g. a corporate DNS server) 
by acting as
+a DNS server for a Hadoop cluster zone/domain. The server is not intended to 
act as a
+primary DNS server and does not forward requests to other servers.
+2. The DNS Server exposes a port that can receive both TCP and UDP requests per
+DNS standards. The default port for DNS protocols is in a restricted, 
administrative port
+range (53), so the port is configurable for deployments in which the service 
may
+not be managed via an administrative account.
+
+##DNS Record Name Structure
+
+The DNS names of generated records are composed from the following elements 
(labels). Note that these elements must be compatible with DNS conventions (see 
“Preferred Name Syntax” in RFC 1035):
+
+* **domain** -​­ the name of the cluster DNS domain. This name is provided 
as a
+configuration property. In addition, it is this name that is configured at a 
parent DNS
+server as the 

[19/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
deleted file mode 100644
index 864a1cf..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.management;
-
-import com.codahale.metrics.JmxReporter;
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.MetricSet;
-import com.codahale.metrics.ScheduledReporter;
-import com.codahale.metrics.Slf4jReporter;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.slider.server.services.workflow.ClosingService;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/**
- * YARN service which hooks up Codahale metrics to 
- * JMX, and, if enabled Ganglia and/or an SLF4J log.
- */
-public class MetricsBindingService extends CompositeService
-implements MetricsKeys {
-  protected static final Logger log =
-  LoggerFactory.getLogger(MetricsBindingService.class);
-  private final MetricRegistry metrics;
-
-  private String reportingDetails = "not started";
-
-
-  public MetricsBindingService(String name,
-  MetricRegistry metrics) {
-super(name);
-Preconditions.checkArgument(metrics != null, "Null metrics");
-this.metrics = metrics;
-  }
-
-  /**
-   * Instantiate...create a metric registry in the process
-   * @param name service name
-   */
-  public MetricsBindingService(String name) {
-this(name, new MetricRegistry());
-  }
-
-  /**
-   * Accessor for the metrics instance
-   * @return the metrics
-   */
-  public MetricRegistry getMetrics() {
-return metrics;
-  }
-
-  @Override
-  protected void serviceStart() throws Exception {
-super.serviceStart();
-
-StringBuilder summary = new StringBuilder();
-Configuration conf = getConfig();
-
-summary.append("Reporting to JMX");
-// always start the JMX binding
-JmxReporter jmxReporter;
-jmxReporter = JmxReporter.forRegistry(metrics).build();
-jmxReporter.start();
-addService(new ClosingService<>(jmxReporter));
-
-
-// Ganglia
-if (conf.getBoolean(METRICS_GANGLIA_ENABLED, false)) {
-  log.warn("Ganglia integration is not implemented");
-/*
-  // This is all disabled due to transitive dependencies on an LGPL library
-  com.codahale.metrics.ganglia.GangliaReporter gangliaReporter;
-  String host = conf.getTrimmed(METRICS_GANGLIA_HOST, "");
-  int port = conf.getInt(METRICS_GANGLIA_PORT, DEFAULT_GANGLIA_PORT);
-  int interval = conf.getInt(METRICS_GANGLIA_REPORT_INTERVAL, 60);
-  int ttl = 1;
-  info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode
-  mcast = 
info.ganglia.gmetric4j.gmetric.GMetric.UDPAddressingMode.getModeForAddress(host);
-  boolean ganglia31 = conf.getBoolean(METRICS_GANGLIA_VERSION_31, true);
-
-  final info.ganglia.gmetric4j.gmetric.GMetric ganglia =
-  new info.ganglia.gmetric4j.gmetric.GMetric(
-  host,
-  port,
-  mcast,
-  ttl,
-  ganglia31);
-  gangliaReporter = 
com.codahale.metrics.ganglia.GangliaReporter.forRegistry(metrics)
-

[20/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
deleted file mode 100644
index 7446e82..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Kill a specific container
- */
-public class ActionKillContainer extends AsyncAction {
-
-  /**
-   *  container to kill
-   */
-  private final ContainerId containerId;
-
-  /**
-   *  handler for the operation
-   */
-  private final RMOperationHandlerActions operationHandler;
-
-  /**
-   * Kill a container
-   * @param containerId container to kill
-   * @param delay
-   * @param timeUnit
-   * @param operationHandler
-   */
-  public ActionKillContainer(
-  ContainerId containerId,
-  long delay,
-  TimeUnit timeUnit,
-  RMOperationHandlerActions operationHandler) {
-super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
-this.operationHandler = operationHandler;
-Preconditions.checkArgument(containerId != null);
-
-this.containerId = containerId;
-  }
-
-  /**
-   * Get the container ID to kill
-   * @return
-   */
-  public ContainerId getContainerId() {
-return containerId;
-  }
-
-  @Override
-  public void execute(SliderAppMaster appMaster,
-  QueueAccess queueService,
-  AppState appState) throws Exception {
-  List opsList = new LinkedList<>();
-ContainerReleaseOperation release = new 
ContainerReleaseOperation(containerId);
-opsList.add(release);
-//now apply the operations
-operationHandler.execute(opsList);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
deleted file mode 100644
index 0d7f7d4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional 

[68/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
new file mode 100644
index 000..3e53418
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionRegistryArgs.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.api.records.ConfigFormat;
+
+import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.ACTION_REGISTRY;
+import static 
org.apache.hadoop.yarn.service.client.params.SliderActions.DESCRIBE_ACTION_REGISTRY;
+import java.io.File;
+
+/**
+ * Registry actions
+ * 
+ * --instance {app name}, if  a / is in it, refers underneath?
+ * --dest {destfile}
+ * --list : list instances of slider service
+ * --listfiles 
+ */
+@Parameters(commandNames = {ACTION_REGISTRY},
+commandDescription = DESCRIBE_ACTION_REGISTRY)
+
+public class ActionRegistryArgs extends AbstractActionArgs {
+
+  public static final String USAGE =
+  "Usage: " + SliderActions.ACTION_REGISTRY
+  + " ("
+  + Arguments.ARG_LIST + "|"
+  + Arguments.ARG_LISTCONF + "|"
+  + Arguments.ARG_LISTEXP + "|"
+  + Arguments.ARG_LISTFILES + "|"
+  + Arguments.ARG_GETCONF + "|"
+  + Arguments.ARG_GETEXP + "> "
+  + Arguments.ARG_NAME + "  "
+  + " )"
+  + "[" + Arguments.ARG_VERBOSE + "] "
+  + "[" + Arguments.ARG_USER + "] "
+  + "[" + Arguments.ARG_OUTPUT + "  ] "
+  + "[" + Arguments.ARG_SERVICETYPE + "  ] "
+  + "[" + Arguments.ARG_FORMAT + " ] "
+  + System.getProperty("line.separator")
+  + "Arguments.ARG_GETEXP only supports " + Arguments.ARG_FORMAT + " json"
+  ;
+  public ActionRegistryArgs() {
+  }
+
+  public ActionRegistryArgs(String name) {
+this.name = name;
+  }
+
+  @Override
+  public String getActionName() {
+return ACTION_REGISTRY;
+  }
+
+  /**
+   * Get the min #of params expected
+   * @return the min number of params in the {@link #parameters} field
+   */
+  @Override
+  public int getMinParams() {
+return 0;
+  }
+  
+  @Parameter(names = {ARG_LIST}, 
+  description = "list services")
+  public boolean list;
+
+  @Parameter(names = {ARG_LISTCONF}, 
+  description = "list configurations")
+  public boolean listConf;
+
+  @Parameter(names = {ARG_GETCONF},
+  description = "get configuration")
+  public String getConf;
+
+  @Parameter(names = {ARG_LISTEXP},
+ description = "list exports")
+  public boolean listExports;
+
+  @Parameter(names = {ARG_GETEXP},
+ description = "get export")
+  public String getExport;
+
+  @Parameter(names = {ARG_LISTFILES},
+  description = "list files")
+  public String listFiles;
+
+  @Parameter(names = {ARG_GETFILES},
+  description = "get files")
+  public String getFiles;
+
+  //--format 
+  @Parameter(names = ARG_FORMAT,
+  description = "Format for a response: ")
+  public String format = ConfigFormat.XML.toString() ;
+
+  @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT, ARG_DEST},
+  description = "Output destination")
+  public File out;
+
+  

[05/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
deleted file mode 100644
index ece65ba..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.history;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoleHistory;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleHistory;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Testing finding nodes for new instances.
- *
- * This stresses the non-AA codepath
- */
-public class TestRoleHistoryFindNodesForNewInstances extends
-BaseMockAppStateTest {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(TestRoleHistoryFindNodesForNewInstances.class);
-
-  public TestRoleHistoryFindNodesForNewInstances() throws BadConfigException {
-  }
-
-  @Override
-  public String getTestName() {
-return "TestFindNodesForNewInstances";
-  }
-
-  private NodeInstance age1Active4;
-  private NodeInstance age2Active2;
-  private NodeInstance age3Active0;
-  private NodeInstance age4Active1;
-  private NodeInstance age2Active0;
-
-  private RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES);
-
-  private RoleStatus roleStat;
-  private RoleStatus roleStat2;
-
-  @Override
-  public void setup() throws Exception {
-super.setup();
-
-age1Active4 = nodeInstance(1, 4, 0, 0);
-age2Active2 = nodeInstance(2, 2, 0, 1);
-age3Active0 = nodeInstance(3, 0, 0, 0);
-age4Active1 = nodeInstance(4, 1, 0, 0);
-age2Active0 = nodeInstance(2, 0, 0, 0);
-
-roleHistory.insert(Arrays.asList(age2Active2, age2Active0, age4Active1,
-age1Active4, age3Active0));
-roleHistory.buildRecentNodeLists();
-
-roleStat = getRole0Status();
-roleStat2 = getRole2Status();
-  }
-
-  public List findNodes(int count) {
-return findNodes(count, roleStat);
-  }
-
-  public List findNodes(int count, RoleStatus roleStatus) {
-List  found = new ArrayList<>();
-for (int i = 0; i < count; i++) {
-  NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus);
-  if (f != null) {
-found.add(f);
-  }
-}
-return found;
-  }
-
-  //@Test
-  public void testFind1NodeR0() throws Throwable {
-NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
-LOG.info("found: {}", found);
-assertTrue(Arrays.asList(age3Active0).contains(found));
-  }
-
-  //@Test
-  public void testFind2NodeR0() throws Throwable {
-NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat);
-LOG.info("found: {}", found);
-assertTrue(Arrays.asList(age2Active0, age3Active0).contains(found));
-NodeInstance found2 = 

[41/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 0ed4860..1049698 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -22,6 +22,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.RetryNTimes;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -40,6 +42,7 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsReque
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
 import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -58,34 +61,32 @@ import 
org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
 import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
 import org.apache.hadoop.yarn.service.ClientAMProtocol;
 import org.apache.hadoop.yarn.service.ServiceMaster;
+import org.apache.hadoop.yarn.service.api.records.Application;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import 
org.apache.hadoop.yarn.service.client.params.AbstractClusterBuildingActionArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionDependencyArgs;
 import org.apache.hadoop.yarn.service.client.params.ActionFlexArgs;
 import org.apache.hadoop.yarn.service.client.params.Arguments;
 import org.apache.hadoop.yarn.service.client.params.ClientArgs;
 import org.apache.hadoop.yarn.service.client.params.CommonArgs;
 import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
 import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
 import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.Times;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.JavaCommandLineBuilder;
-import org.apache.slider.core.registry.SliderRegistryUtils;
-import org.apache.slider.core.zk.ZKIntegration;
-import org.apache.slider.core.zk.ZookeeperUtils;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.exceptions.UsageException;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
+import 

[28/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
deleted file mode 100644
index 64fd8ae..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java
+++ /dev/null
@@ -1,611 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.w3c.dom.Document;
-import org.xml.sax.SAXException;
-
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.parsers.ParserConfigurationException;
-import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.StringWriter;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- * 
- * Some of the methods take an argument of a map iterable for their sources; 
this allows
- * the same method
- */
-public class ConfigHelper {
-  private static final Logger log = 
LoggerFactory.getLogger(ConfigHelper.class);
-
-  /**
-   * Dump the (sorted) configuration
-   * @param conf config
-   * @return the sorted keyset
-   */
-  public static Set dumpConf(Configuration conf) {
-Set keys = sortedConfigKeys(conf);
-for (String key : keys) {
-  log.info("{}={}", key, conf.get(key));
-}
-return keys;
-  }
-
-  /**
-   * Take a configuration and return a sorted set
-   * @param conf config
-   * @return the sorted keyset
-
-   */
-  public static Set sortedConfigKeys(Iterable> conf) {
-TreeSet sorted = new TreeSet();
-for (Map.Entry entry : conf) {
-  sorted.add(entry.getKey());
-}
-return sorted;
-  }
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Map map,
-  String origin) throws BadConfigException {
-addConfigMap(config, map.entrySet(), origin);
-  }
-  
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Iterable> map,
-  String origin) throws BadConfigException {
-for (Map.Entry mapEntry : map) {
-  String key = mapEntry.getKey();
-  String value = mapEntry.getValue();
-  if (value == null) {
-throw new BadConfigException("Null value for property " + key);
-  }
-  

[09/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
deleted file mode 100644
index 7b0586e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
-import org.apache.slider.core.exceptions.UsageException;
-import org.apache.slider.utils.SliderTestBase;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-
-/**
- * Test the argument parsing/validation logic.
- */
-public class TestClientBadArgs extends SliderTestBase {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(TestClientBadArgs.class);
-
-  //@Test
-  public void testNoAction() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: slider COMMAND",
- EMPTY_LIST);
-
-  }
-
-  //@Test
-  public void testUnknownAction() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "not-a-known-action",
- Arrays.asList("not-a-known-action"));
-  }
-
-  //@Test
-  public void testActionWithoutOptions() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Usage: slider build ",
- Arrays.asList(SliderActions.ACTION_BUILD));
-  }
-
-  //@Test
-  public void testActionWithoutEnoughArgs() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
- Arrays.asList(SliderActions.ACTION_START));
-  }
-
-  //@Test
-  public void testActionWithTooManyArgs() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
- Arrays.asList(SliderActions.ACTION_HELP,
- "hello, world"));
-  }
-
-  //@Test
-  public void testBadImageArg() throws Throwable {
-launchExpectingException(SliderClient.class,
- createTestConfig(),
- "Unknown option: --image",
-Arrays.asList(SliderActions.ACTION_HELP,
- Arguments.ARG_IMAGE));
-  }
-
-  //@Test
-  public void testRegistryUsage() throws Throwable {
-Throwable exception = launchExpectingException(SliderClient.class,
-createTestConfig(),
-"org.apache.slider.core.exceptions.UsageException: Argument --name " +
-"missing",
-Arrays.asList(SliderActions.ACTION_REGISTRY));
-assertTrue(exception instanceof UsageException);
-LOG.info(exception.toString());
-  }
-
-  //@Test
-  public void testRegistryExportBadUsage1() throws Throwable {
-

[36/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
new file mode 100644
index 000..c0712c3
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/KerberosDiags.java
@@ -0,0 +1,680 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslPropertiesResolver;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.crypto.Cipher;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.InvocationTargetException;
+import java.net.InetAddress;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.security.UserGroupInformation.*;
+import static org.apache.hadoop.security.authentication.util.KerberosUtil.*;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+
+/**
+ * Kerberos diagnostics
+ * At some point this may move to hadoop core, so please keep use of slider
+ * methods and classes to ~0.
+ *
+ * This operation expands some of the diagnostic output of the security code,
+ * but not all. For completeness
+ *
+ * Set the environment variable {@code HADOOP_JAAS_DEBUG=true}
+ * Set the log level for {@code org.apache.hadoop.security=DEBUG}
+ */
+public class KerberosDiags implements Closeable {
+
+  private static final Logger LOG = 
LoggerFactory.getLogger(KerberosDiags.class);
+  public static final String KRB5_CCNAME = "KRB5CCNAME";
+  public static final String JAVA_SECURITY_KRB5_CONF
+= "java.security.krb5.conf";
+  public static final String JAVA_SECURITY_KRB5_REALM
+= "java.security.krb5.realm";
+  public static final String SUN_SECURITY_KRB5_DEBUG
+= "sun.security.krb5.debug";
+  public static final String SUN_SECURITY_SPNEGO_DEBUG
+= "sun.security.spnego.debug";
+  public static final String SUN_SECURITY_JAAS_FILE
+= "java.security.auth.login.config";
+  public static final String KERBEROS_KINIT_COMMAND
+= "hadoop.kerberos.kinit.command";
+  public static final String HADOOP_AUTHENTICATION_IS_DISABLED
+  = "Hadoop authentication is disabled";
+  public static final String UNSET = "(unset)";
+  public static final String NO_DEFAULT_REALM = "Cannot locate default realm";
+
+  private final Configuration conf;
+  private final List services;
+  private final PrintStream out;
+  private final File keytab;
+  private final String principal;
+  private final long minKeyLength;
+  private final boolean securityRequired;
+
+  public static final String CAT_JVM = "JVM";
+  public static final String CAT_JAAS = "JAAS";
+  public static final String CAT_CONFIG = "CONFIG";
+  public static final String CAT_LOGIN = "LOGIN";
+  public static final String CAT_KERBEROS = "KERBEROS";
+  public static 

[48/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
deleted file mode 100644
index 8310530..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestLoadExampleAppJson extends Assert {
-  private String resource;
-
-  public TestLoadExampleAppJson(String resource) {
-this.resource = resource;
-  }
-
-  @Parameterized.Parameters
-  public static Collection filenames() {
-String[][] stringArray = new String[ExampleAppJson
-.ALL_EXAMPLE_RESOURCES.size()][1];
-int i = 0;
-for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
-  stringArray[i++][0] = s;
-}
-return Arrays.asList(stringArray);
-  }
-
-  @Test
-  public void testLoadResource() throws Throwable {
-try {
-  Application application = JSON_SER_DESER.fromResource(resource);
-
-  SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-  FileSystem mockFs = createNiceMock(FileSystem.class);
-  expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-  expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-  new Path("cluster_dir_path")).anyTimes();
-  replay(sfs, mockFs);
-
-  ServiceApiUtil.validateAndResolveApplication(application, sfs,
-  new YarnConfiguration());
-} catch (Exception e) {
-  throw new Exception("exception loading " + resource + ":" + 
e.toString());
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
deleted file mode 100644
index 98c78d3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you 

[39/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
new file mode 100644
index 000..e4eae20
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static 
org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+
+/**
+ * Launcher of applications: base class
+ */
+public class AbstractLauncher {
+  private static final Logger log =
+LoggerFactory.getLogger(AbstractLauncher.class);
+  public static final String CLASSPATH = "CLASSPATH";
+  /**
+   * Filesystem to use for the launch
+   */
+  protected final CoreFileSystem coreFileSystem;
+  /**
+   * Env vars; set up at final launch stage
+   */
+  protected final Map envVars = new HashMap<>();
+  protected final ContainerLaunchContext containerLaunchContext =
+Records.newRecord(ContainerLaunchContext.class);
+  protected final List commands = new ArrayList<>(20);
+  protected final Map localResources = new HashMap<>();
+  protected final Map mountPaths = new HashMap<>();
+  private final Map serviceData = new HashMap<>();
+  // security
+  protected final Credentials credentials;
+  protected boolean yarnDockerMode = false;
+  protected String dockerImage;
+  protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+  protected String dockerHostname;
+  protected String runPrivilegedContainer;
+
+
+  /**
+   * Create instance.
+   * @param coreFileSystem filesystem
+   * @param credentials initial set of credentials -null is permitted
+   */
+  public AbstractLauncher(
+  CoreFileSystem coreFileSystem,
+  Credentials credentials) {
+this.coreFileSystem = coreFileSystem;
+this.credentials = credentials != null ? credentials: new Credentials();
+  }
+  
+  public void setYarnDockerMode(boolean yarnDockerMode){
+this.yarnDockerMode = yarnDockerMode;
+  }
+
+  /**
+   * Get the env vars to work on
+   * @return env vars
+   */
+  public Map getEnv() {
+return envVars;
+  }
+
+  /**
+   * Get the launch commands.
+   * @return the live list of commands 
+   */
+  public List getCommands() {
+return commands;
+  }
+
+  public void addLocalResource(String subPath, LocalResource resource) {
+localResources.put(subPath, resource);
+  }
+
+  public void addLocalResource(String subPath, LocalResource resource, String 
mountPath) {
+localResources.put(subPath, 

[72/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 88f74ef..17f8c95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -22,7 +22,7 @@ info:
 Bringing a new service on YARN today is not a simple experience. The APIs 
of existing frameworks are either too low level (native YARN), require writing 
new code (for frameworks with programmatic APIs) or writing a complex spec (for 
declarative frameworks). In addition to building critical building blocks 
inside YARN (as part of other efforts at 
link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a 
need for simplifying the user facing story for building services. Experience of 
projects like Apache Slider running real-life services like HBase, Storm, 
Accumulo, Solr etc, gives us some very good insights on how simplified APIs for 
services should look like.
 
 
-To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component apps to 
complex multi-component assemblies needing orchestration.
+To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component service to 
complex multi-component assemblies needing orchestration.
 
 
 We should also look at making this a unified REST based entry point for 
other important features like resource-profile management 
(link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), 
package-definitions' lifecycle-management and service-discovery 
(link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]).
 We also need to flesh out its relation to our present much lower level REST 
APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN 
for application-submission and management.
@@ -41,177 +41,177 @@ schemes:
   - http
   - https
 # will be prefixed to all paths
-basePath: /services/v1/
+basePath: /ws/v1/
 consumes:
   - application/json
 produces:
   - application/json
 paths:
-  /applications:
+  /services:
 get:
-  summary: List of applications/services running in the cluster
-  description: Get a list of all currently running applications (response 
includes a minimal projection of the application info). For more details do a 
GET on a specific application name.
+  summary: List of services running in the cluster
+  description: Get a list of all currently running services (response 
includes a minimal projection of the service info). For more details do a GET 
on a specific service name.
   responses:
 200:
-  description: An array of applications
+  description: An array of services
   schema:
 type: array
 items:
-  $ref: '#/definitions/Application'
+  $ref: '#/definitions/Service'
 default:
   description: Unexpected error
   schema:
-$ref: '#/definitions/ApplicationStatus'
+$ref: '#/definitions/ServiceStatus'
 post:
-  summary: Create an application/service
-  description: Create an application. The request JSON is an Application 
object with details required for creation. If the request is successful it 
returns 202 Accepted. A success of this API only confirms success in submission 
of the application creation request. There is no guarantee that the application 
will actually reach a RUNNING state. Resource availability and several other 
factors determines if the application will be deployed in the cluster. It is 
expected that clients would subsequently call the GET API to get details of the 
application and determine its state.
+  summary: Create a service
+  description: Create a service. The request JSON is a service object with 
details required for creation. If the request is successful it returns 202 
Accepted. A success of this API only 

[73/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

YARN-7091. Rename application to service in yarn-native-services. Contributed 
by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db5888ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db5888ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db5888ea

Branch: refs/heads/yarn-native-services
Commit: db5888ea20a0765f5311d038afbbc78c218679bd
Parents: 963f2d9
Author: Billie Rinaldi 
Authored: Mon Aug 28 09:59:55 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:45:12 2017 -0700

--
 .../resources/assemblies/hadoop-yarn-dist.xml   |8 +-
 .../assemblies/hadoop-yarn-services-dist.xml|2 +-
 hadoop-project/pom.xml  |2 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   14 +-
 .../hadoop-yarn/conf/yarn-env.sh|2 +-
 .../hadoop-yarn-services-api/pom.xml|2 +-
 .../hadoop/yarn/service/webapp/ApiServer.java   |  274 +++
 .../yarn/service/webapp/ApiServerWebApp.java|  159 ++
 .../service/webapp/ApplicationApiService.java   |  275 ---
 .../service/webapp/ApplicationApiWebApp.java|  123 --
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  185 +-
 .../src/main/scripts/run_rest_service.sh|   28 -
 .../dev-support/findbugs-exclude.xml|   48 +
 .../conf/yarnservice-log4j.properties   |   62 +
 .../hadoop-yarn-services-core/pom.xml   |  408 +
 .../hadoop/yarn/service/ClientAMProtocol.java   |   40 +
 .../hadoop/yarn/service/ClientAMService.java|  132 ++
 .../yarn/service/ContainerFailureTracker.java   |   89 +
 .../hadoop/yarn/service/ServiceContext.java |   41 +
 .../hadoop/yarn/service/ServiceMaster.java  |  156 ++
 .../hadoop/yarn/service/ServiceMetrics.java |   98 +
 .../hadoop/yarn/service/ServiceScheduler.java   |  654 +++
 .../yarn/service/api/ServiceApiConstants.java   |   69 +
 .../yarn/service/api/records/Artifact.java  |  160 ++
 .../yarn/service/api/records/BaseResource.java  |   52 +
 .../yarn/service/api/records/Component.java |  412 +
 .../yarn/service/api/records/ConfigFile.java|  225 +++
 .../yarn/service/api/records/ConfigFormat.java  |   67 +
 .../yarn/service/api/records/Configuration.java |  225 +++
 .../yarn/service/api/records/Container.java |  297 +++
 .../service/api/records/ContainerState.java |   30 +
 .../hadoop/yarn/service/api/records/Error.java  |  129 ++
 .../service/api/records/PlacementPolicy.java|  102 ++
 .../service/api/records/ReadinessCheck.java |  175 ++
 .../yarn/service/api/records/Resource.java  |  159 ++
 .../yarn/service/api/records/Service.java   |  466 +
 .../yarn/service/api/records/ServiceState.java  |   33 +
 .../yarn/service/api/records/ServiceStatus.java |  148 ++
 .../yarn/service/client/ClientAMProxy.java  |   57 +
 .../hadoop/yarn/service/client/ServiceCLI.java  |  112 ++
 .../yarn/service/client/ServiceClient.java  |  892 +
 .../client/params/AbstractActionArgs.java   |  158 ++
 .../client/params/AbstractArgsDelegate.java |   28 +
 .../AbstractClusterBuildingActionArgs.java  |   58 +
 .../service/client/params/ActionBuildArgs.java  |   31 +
 .../service/client/params/ActionClientArgs.java |   71 +
 .../service/client/params/ActionCreateArgs.java |   33 +
 .../client/params/ActionDependencyArgs.java |   65 +
 .../client/params/ActionDestroyArgs.java|   37 +
 .../service/client/params/ActionExistsArgs.java |   49 +
 .../service/client/params/ActionFlexArgs.java   |   50 +
 .../service/client/params/ActionFreezeArgs.java |   56 +
 .../service/client/params/ActionHelpArgs.java   |   44 +
 .../service/client/params/ActionKDiagArgs.java  |   76 +
 .../service/client/params/ActionKeytabArgs.java |   76 +
 .../service/client/params/ActionListArgs.java   |   76 +
 .../client/params/ActionRegistryArgs.java   |  218 +++
 .../client/params/ActionResolveArgs.java|  153 ++
 .../client/params/ActionResourceArgs.java   |   70 +
 .../service/client/params/ActionStatusArgs.java |   51 +
 .../service/client/params/ActionThawArgs.java   |   67 +
 .../service/client/params/ActionTokensArgs.java |   78 +
 .../service/client/params/ActionUpdateArgs.java |   32 +
 .../yarn/service/client/params/ArgOps.java  |  156 ++
 .../yarn/service/client/params/Arguments.java   |  103 ++
 .../yarn/service/client/params/ClientArgs.java  |  252 +++
 .../yarn/service/client/params/CommonArgs.java  |  282 +++
 .../client/params/ComponentArgsDelegate.java|   52 +
 .../client/params/DontSplitArguments.java   |   34 +
 .../client/params/LaunchArgsAccessor.java   |   30 +
 .../client/params/LaunchArgsDelegate.java   |   51 +
 .../client/params/OptionArgsDelegate.java   |   66 +
 .../client/params/PathArgumentConverter.java|   34 +
 

[65/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
new file mode 100644
index 000..684f655
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/MonitorUtils.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.monitor.probe;
+
+import org.apache.hadoop.yarn.service.api.records.ReadinessCheck;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Formatter;
+import java.util.Locale;
+
+/**
+ * Various utils to work with the monitor
+ */
+public final class MonitorUtils {
+  protected static final Logger LOG = LoggerFactory.getLogger(MonitorUtils
+  .class);
+
+  private MonitorUtils() {
+  }
+
+  public static String toPlural(int val) {
+return val != 1 ? "s" : "";
+  }
+
+  /**
+   * Convert milliseconds to human time -the exact format is unspecified
+   * @param milliseconds a time in milliseconds
+   * @return a time that is converted to human intervals
+   */
+  public static String millisToHumanTime(long milliseconds) {
+StringBuilder sb = new StringBuilder();
+// Send all output to the Appendable object sb
+Formatter formatter = new Formatter(sb, Locale.US);
+
+long s = Math.abs(milliseconds / 1000);
+long m = Math.abs(milliseconds % 1000);
+if (milliseconds > 0) {
+  formatter.format("%d.%03ds", s, m);
+} else if (milliseconds == 0) {
+  formatter.format("0");
+} else {
+  formatter.format("-%d.%03ds", s, m);
+}
+return sb.toString();
+  }
+
+  public static Probe getProbe(ReadinessCheck readinessCheck) {
+if (readinessCheck == null) {
+  return null;
+}
+if (readinessCheck.getType() == null) {
+  return null;
+}
+try {
+  switch (readinessCheck.getType()) {
+  case HTTP:
+return HttpProbe.create(readinessCheck.getProps());
+  case PORT:
+return PortProbe.create(readinessCheck.getProps());
+  default:
+return null;
+  }
+} catch (Throwable t) {
+  throw new IllegalArgumentException("Error creating readiness check " +
+  t);
+}
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
new file mode 100644
index 000..aba5859
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/monitor/probe/PortProbe.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You 

[61/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
new file mode 100644
index 000..be36335
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_COMPONENT_NAME;
+import static 
org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIMITED_LIFETIME;
+import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for ServiceApiUtil helper methods.
+ */
+public class TestServiceApiUtil {
+  private static final Logger LOG = LoggerFactory
+  .getLogger(TestServiceApiUtil.class);
+  private static final String EXCEPTION_PREFIX = "Should have thrown " +
+  "exception: ";
+  private static final String NO_EXCEPTION_PREFIX = "Should not have thrown " +
+  "exception: ";
+
+  private static final String LEN_64_STR =
+  "abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01";
+
+  private static final YarnConfiguration CONF_DEFAULT_DNS = new
+  YarnConfiguration();
+  private static final YarnConfiguration CONF_DNS_ENABLED = new
+  YarnConfiguration();
+
+  @BeforeClass
+  public static void init() {
+CONF_DNS_ENABLED.setBoolean(RegistryConstants.KEY_DNS_ENABLED, true);
+  }
+
+  @Test(timeout = 9)
+  public void testResourceValidation() throws Exception {
+assertEquals(RegistryConstants.MAX_FQDN_LABEL_LENGTH + 1, LEN_64_STR
+.length());
+
+SliderFileSystem sfs = initMock(null);
+
+Service app = new Service();
+
+// no name
+try {
+  ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+  Assert.fail(EXCEPTION_PREFIX + "service with no name");
+} catch (IllegalArgumentException e) {
+  assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
+}
+
+// bad format name
+String[] badNames = {"4finance", "Finance", "finance@home", LEN_64_STR};
+for (String badName : badNames) {
+  app.setName(badName);
+  try {
+ServiceApiUtil.validateAndResolveService(app, sfs, CONF_DNS_ENABLED);
+Assert.fail(EXCEPTION_PREFIX + "service with bad name " + badName);
+  } catch (IllegalArgumentException e) {
+

[04/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
deleted file mode 100644
index 9632265..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderInternalStateException;
-import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.ContainerOutcome;
-import org.apache.slider.server.appmaster.state.NodeEntry;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.NodeMap;
-import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.slider.utils.SliderTestBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map.Entry;
-
-/**
- * Base for app state tests.
- */
-public abstract class BaseMockAppStateTest extends SliderTestBase implements
-MockRoles {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(BaseMockAppStateTest.class);
-  protected static final List EMPTY_ID_LIST = Collections
-  .emptyList();
-
-  protected final MockFactory factory = MockFactory.INSTANCE;
-  protected MockAppState appState;
-  protected MockYarnEngine engine;
-  protected FileSystem fs;
-  protected SliderFileSystem sliderFileSystem;
-  protected File historyWorkDir;
-  protected Path historyPath;
-  protected MockApplicationId applicationId;
-  protected 

[11/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
deleted file mode 100644
index 9c653f3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCompositeService.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * An extended composite service which stops itself if any child service
- * fails, or when all its children have successfully stopped without failure.
- *
- * Lifecycle
- * 
- *   If any child exits with a failure: this service stops, propagating
- *   the exception.
- *   When all child services has stopped, this service stops itself
- * 
- *
- */
-public class WorkflowCompositeService extends CompositeService
-implements ServiceParent, ServiceStateChangeListener {
-
-  private static final Logger LOG =
-LoggerFactory.getLogger(WorkflowCompositeService.class);
-
-  /**
-   * Deadlock-avoiding overridden config for slider services; see SLIDER-1052
-   */
-  private volatile Configuration configuration;
-
-  /**
-   * Construct an instance
-   * @param name name of this service instance
-   */
-  public WorkflowCompositeService(String name) {
-super(name);
-  }
-
-  @Override
-  public Configuration getConfig() {
-return configuration;
-  }
-
-  @Override
-  protected void setConfig(Configuration conf) {
-super.setConfig(conf);
-configuration = conf;
-  }
-
-  /**
-   * Construct an instance with the default name.
-   */
-  public WorkflowCompositeService() {
-this("WorkflowCompositeService");
-  }
-
-  /**
-   * Varargs constructor
-   * @param name name of this service instance
-   * @param children children
-   */
-  public WorkflowCompositeService(String name, Service... children) {
-this(name);
-for (Service child : children) {
-  addService(child);
-}
-  }
-
-  /**
-   * Construct with a list of children
-   * @param name name of this service instance
-   * @param children children to add
-   */
-  public WorkflowCompositeService(String name, List children) {
-this(name);
-for (Service child : children) {
-  addService(child);
-}
-  }
-
-  /**
-   * Add a service, and register it
-   * @param service the {@link Service} to be added.
-   * Important: do not add a service to a parent during your own 
serviceInit/start,
-   * in Hadoop 2.2; you will trigger a ConcurrentModificationException.
-   */
-  @Override
-  public synchronized void addService(Service service) {
-Preconditions.checkArgument(service != null, "null service argument");
-service.registerServiceListener(this);
-super.addService(service);
-  }
-
-  /**
-   * When this service is started, any service stopping with a failure
-   * exception is converted immediately into a failure of this service, 
-   * storing the failure and stopping ourselves.
-   * @param child the service that has changed.
-   */
-  @Override
-  public void stateChanged(Service child) {
-//if that child stopped while we are running:
-if (isInState(STATE.STARTED) && child.isInState(STATE.STOPPED)) {
-  // a child service 

[31/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
deleted file mode 100644
index 7712191..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ /dev/null
@@ -1,2783 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.slider.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathNotFoundException;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.RegistryPathStatus;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.KerberosDiags;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
-import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.YarnClientApplication;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.client.ipc.SliderClusterOperations;
-import org.apache.slider.common.Constants;
-import 

[40/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
new file mode 100644
index 000..e310f45
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ActionUpdateArgs.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client.params;
+
+import com.beust.jcommander.Parameters;
+
+@Parameters(commandNames = { SliderActions.ACTION_UPDATE},
+commandDescription = SliderActions.DESCRIBE_ACTION_UPDATE)
+
+public class ActionUpdateArgs extends AbstractClusterBuildingActionArgs {
+
+  @Override
+  public String getActionName() {
+return SliderActions.ACTION_UPDATE;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
index f7b7349..00151f4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/ArgOps.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.service.client.params;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.ErrorStrings;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
index d8d8ab4..204149b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/client/params/Arguments.java
@@ -26,59 +26,38 @@ package org.apache.hadoop.yarn.service.client.params;
  * so it is easier to see what arguments are there
  */
 public interface Arguments {
-  String ARG_ADDON = "--addon";
-  String 

[51/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
deleted file mode 100644
index fe8cce8..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ConfigHelper.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.StringWriter;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * Methods to aid in config, both in the Configuration class and
- * with other parts of setting up Slider-initated processes.
- *
- * Some of the methods take an argument of a map iterable for their sources; 
this allows
- * the same method
- */
-public class ConfigHelper {
-  private static final Logger log = 
LoggerFactory.getLogger(ConfigHelper.class);
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Map map,
-  String origin) throws BadConfigException {
-addConfigMap(config, map.entrySet(), origin);
-  }
-
-  /**
-   * Set an entire map full of values
-   *
-   * @param config config to patch
-   * @param map map of data
-   * @param origin origin data
-   */
-  public static void addConfigMap(Configuration config,
-  Iterable> map,
-  String origin) throws BadConfigException {
-for (Map.Entry mapEntry : map) {
-  String key = mapEntry.getKey();
-  String value = mapEntry.getValue();
-  if (value == null) {
-throw new BadConfigException("Null value for property " + key);
-  }
-  config.set(key, value, origin);
-}
-  }
-
-  /**
-   * Convert to an XML string
-   * @param conf configuration
-   * @return conf
-   * @throws IOException
-   */
-  public static String toXml(Configuration conf) throws IOException {
-StringWriter writer = new StringWriter();
-conf.writeXml(writer);
-return writer.toString();
-  }
-
-
-  /**
-   * Register a resource as a default resource.
-   * Do not attempt to use this unless you understand that the
-   * order in which default resources are loaded affects the outcome,
-   * and that subclasses of Configuration often register new default
-   * resources
-   * @param resource the resource name
-   * @return the URL or null
-   */
-  public static URL registerDefaultResource(String resource) {
-URL resURL = getResourceUrl(resource);
-if (resURL != null) {
-  Configuration.addDefaultResource(resource);
-}
-return resURL;
-  }
-
-  /**
-   * Load a configuration from a resource on this classpath.
-   * If the resource is not found, an empty configuration is returned
-   * @param resource the resource name
-   * @return the loaded configuration.
-   */
-  public static Configuration loadFromResource(String resource) {
-Configuration conf = new Configuration(false);
-URL resURL = getResourceUrl(resource);
-if (resURL != null) {
-  log.debug("loaded resources from {}", resURL);
-  conf.addResource(resource);
-} else{
-  log.debug("failed to find {} on 

[14/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
deleted file mode 100644
index 0cac430..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import com.codahale.metrics.servlets.HealthCheckServlet;
-import com.codahale.metrics.servlets.MetricsServlet;
-import com.codahale.metrics.servlets.PingServlet;
-import com.codahale.metrics.servlets.ThreadDumpServlet;
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.container.filter.GZIPContentEncodingFilter;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.spi.container.servlet.ServletContainer;
-import org.apache.hadoop.yarn.webapp.Dispatcher;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.slider.server.appmaster.web.rest.AMWadlGeneratorConfig;
-import org.apache.slider.server.appmaster.web.rest.AMWebServices;
-import 
org.apache.slider.server.appmaster.web.rest.SliderJacksonJaxbJsonProvider;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-
-/**
- * 
- */
-public class SliderAMWebApp extends WebApp {
-  public static final String BASE_PATH = "slideram";
-  public static final String CONTAINER_STATS = "/stats";
-  public static final String CLUSTER_SPEC = "/spec";
-
-  private final WebAppApi webAppApi;
-
-  public SliderAMWebApp(WebAppApi webAppApi) {
-Preconditions.checkArgument(webAppApi != null, "webAppApi null");
-this.webAppApi = webAppApi;
-  }
-
-  @Override
-  public void setup() {
-Logger.getLogger("com.sun.jersey").setLevel(Level.FINEST);
-// Make one of these to ensure that the jax-b annotations
-// are properly picked up.
-bind(SliderJacksonJaxbJsonProvider.class);
-
-// Get exceptions printed to the screen
-bind(GenericExceptionHandler.class);
-// bind the REST interface
-bind(AMWebServices.class);
-
-//bind(AMAgentWebServices.class);
-route("/", SliderAMController.class);
-route(CONTAINER_STATS, SliderAMController.class, "containerStats");
-route(CLUSTER_SPEC, SliderAMController.class, "specification");
-  }
-
-  @Override
-  public void configureServlets() {
-setup();
-
-serve("/", "/__stop").with(Dispatcher.class);
-
-for (String path : this.getServePathSpecs()) {
-  serve(path).with(Dispatcher.class);
-}
-
-serve(SYSTEM_HEALTHCHECK)
-.with(new HealthCheckServlet(new HealthCheckRegistry()));
-serve(SYSTEM_METRICS).with(new MetricsServlet(new MetricRegistry()));
-serve(SYSTEM_PING).with(new PingServlet());
-serve(SYSTEM_THREADS).with(new ThreadDumpServlet());
-
-String regex = "(?!/ws)";
-serveRegex(regex).with(SliderDefaultWrapperServlet.class); 
-
-Map params = new HashMap<>();
-params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
-params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
-params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
-params.put(ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS, 

[49/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
deleted file mode 100644
index 1fa07ce..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ZookeeperUtils.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ZookeeperUtils {
-  public static final int DEFAULT_PORT = 2181;
-
-  public static String buildConnectionString(String zkHosts, int port) {
-String zkPort = Integer.toString(port);
-//parse the hosts
-String[] hostlist = zkHosts.split(",", 0);
-String quorum = SliderUtils.join(hostlist, ":" + zkPort + ",", false);
-return quorum;
-  }
-
-  /**
-   * Take a quorum list and split it to (trimmed) pairs
-   * @param hostPortQuorumList list of form h1:port, h2:port2,...
-   * @return a possibly empty list of values between commas. They may not be
-   * valid hostname:port pairs
-   */
-  public static List splitToPairs(String hostPortQuorumList) {
-// split an address hot
-String[] strings = StringUtils.getStrings(hostPortQuorumList);
-int len = 0;
-if (strings != null) {
-  len = strings.length;
-}
-List tuples = new ArrayList(len);
-if (strings != null) {
-  for (String s : strings) {
-tuples.add(s.trim());
-  }
-}
-return tuples;
-  }
-
-  /**
-   * Split a quorum list into a list of hostnames and ports
-   * @param hostPortQuorumList split to a list of hosts and ports
-   * @return a list of values
-   */
-  public static List splitToHostsAndPorts(String 
hostPortQuorumList) {
-// split an address hot
-String[] strings = StringUtils.getStrings(hostPortQuorumList);
-int len = 0;
-if (strings != null) {
-  len = strings.length;
-}
-List list = new ArrayList(len);
-if (strings != null) {
-  for (String s : strings) {
-
list.add(HostAndPort.fromString(s.trim()).withDefaultPort(DEFAULT_PORT));
-  }
-}
-return list;
-  }
-
-  /**
-   * Build up to a hosts only list
-   * @param hostAndPorts
-   * @return a list of the hosts only
-   */
-  public static String buildHostsOnlyList(List hostAndPorts) {
-StringBuilder sb = new StringBuilder();
-for (HostAndPort hostAndPort : hostAndPorts) {
-  sb.append(hostAndPort.getHostText()).append(",");
-}
-if (sb.length() > 0) {
-  sb.delete(sb.length() - 1, sb.length());
-}
-return sb.toString();
-  }
-
-  public static String buildQuorumEntry(HostAndPort hostAndPort,
-int defaultPort) {
-String s = hostAndPort.toString();
-if (hostAndPort.hasPort()) {
-  return s;
-} else {
-  return s + ":" + defaultPort;
-}
-  }
-
-  /**
-   * Build a quorum list, injecting a ":defaultPort" ref if needed on
-   * any entry without one
-   * @param hostAndPorts
-   * @param defaultPort
-   * @return
-   */
-  public static String buildQuorum(List hostAndPorts, int 
defaultPort) {
-List entries = new ArrayList(hostAndPorts.size());
-for (HostAndPort hostAndPort : hostAndPorts) {
-  entries.add(buildQuorumEntry(hostAndPort, defaultPort));
-}
-return SliderUtils.join(entries, ",", false);
-  }
-  
-  public static String convertToHostsOnlyList(String quorum) throws
-  BadConfigException {
-List 

[34/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
deleted file mode 100644
index 2f71004..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api;
-
-/**
- * These are the keys valid in resource options
- *
- /*
-
- Container failure window.
-
- The window is calculated in minutes as as (days * 24 *60 + hours* 24 + 
minutes)
-
- Every interval of this period after the AM is started/restarted becomes
- the time period in which the CONTAINER_FAILURE_THRESHOLD value is calculated.
- 
- After the window limit is reached, the failure counts are reset. This
- is not a sliding window/moving average policy, simply a rule such as
- "every six hours the failure count is reset"
-
-
- 
- ===
- 
-
- */
-public interface ResourceKeys {
-
-
-  /**
-   * #of instances of a component: {@value}
-   *
-  */
-  String COMPONENT_INSTANCES = "yarn.component.instances";
-
-  /**
-   * Whether to use unique names for each instance of a component: {@value}
-   */
-  String UNIQUE_NAMES = "component.unique.names";
-
-  /**
-   *  Amount of memory to ask YARN for in MB.
-   *  Important: this may be a hard limit on the
-   *  amount of RAM that the service can use
-   *  {@value}
-   */
-  String YARN_MEMORY = "yarn.memory";
-  
-  /** {@value} */
-  int DEF_YARN_MEMORY = 256;
-  
-  /**
-   * Number of cores/virtual cores to ask YARN for
-   *  {@value}
-   */
-  String YARN_CORES = "yarn.vcores";
-
-  /**
-   * If normalization is set to false, then if the resource (memory and/or
-   * vcore) requested by a role is higher than YARN limits, then the resource
-   * request is not normalized. If this causes failures at the YARN level then
-   * applications are expecting that to happen. Default value is true.
-   */
-  String YARN_RESOURCE_NORMALIZATION_ENABLED =
-  "yarn.resource.normalization.enabled";
-
-  /**
-   * Number of disks per instance to ask YARN for
-   *  {@value}
-   */
-  String YARN_DISKS = "yarn.disks.count-per-instance";
-
-  /**
-   * Disk size per disk to ask YARN for
-   *  {@value}
-   */
-  String YARN_DISK_SIZE = "yarn.disk.size";
-
-  /** {@value} */
-  int DEF_YARN_CORES = 1;
-
-
-  /**
-   * Label expression that this container must satisfy
-   *  {@value}
-   */
-  String YARN_LABEL_EXPRESSION = "yarn.label.expression";
-
-  /** default label expression: */
-  String DEF_YARN_LABEL_EXPRESSION = null;
-
-
-  /**
-   * Constant to indicate that the requirements of a YARN resource limit
-   * (cores, memory, ...) should be set to the maximum allowed by
-   * the queue into which the YARN container requests are placed.
-   */
-  String YARN_RESOURCE_MAX = "max";
-  
-  /**
-   * Mandatory property for all roles
-   * 1. this must be defined.
-   * 2. this must be >= 1
-   * 3. this must not match any other role priority in the cluster.
-   */
-  String COMPONENT_PRIORITY = "yarn.role.priority";
-  
-  /**
-   * placement policy
-   */
-  String COMPONENT_PLACEMENT_POLICY = "yarn.component.placement.policy";
-
-  /**
-   * Maximum number of node failures that can be tolerated by a component on a 
specific node
-   */
-  String NODE_FAILURE_THRESHOLD =
-  "yarn.node.failure.threshold";
-
-  /**
-   * maximum number of failed containers (in a single role)
-   * before the cluster is deemed to have failed {@value}
-   */
-  String CONTAINER_FAILURE_THRESHOLD =
-  

[02/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
deleted file mode 100644
index a667432..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/TestWorkflowServiceTerminatingRunnable.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.junit.Test;
-
-
-public class TestWorkflowServiceTerminatingRunnable extends 
WorkflowServiceTestBase {
-
-  //@Test
-  public void testNoservice() throws Throwable {
-
-try {
-  new ServiceTerminatingRunnable(null, new SimpleRunnable());
-  fail("unexpected ");
-} catch (IllegalArgumentException e) {
-
-  // expected
-}
-  }
-
-
-  //@Test
-  public void testBasicRun() throws Throwable {
-
-WorkflowCompositeService svc = run(new WorkflowCompositeService());
-ServiceTerminatingRunnable runnable = new ServiceTerminatingRunnable(svc,
-new SimpleRunnable());
-
-// synchronous in-thread execution
-runnable.run();
-assertStopped(svc);
-  }
-
-  //@Test
-  public void testFailureRun() throws Throwable {
-
-WorkflowCompositeService svc = run(new WorkflowCompositeService());
-ServiceTerminatingRunnable runnable =
-new ServiceTerminatingRunnable(svc, new SimpleRunnable(true));
-
-// synchronous in-thread execution
-runnable.run();
-assertStopped(svc);
-assertNotNull(runnable.getException());
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
deleted file mode 100644
index f38bd9d..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/services/workflow/WorkflowServiceTestBase.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.services.workflow;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.Service;
-import org.junit.Assert;

[12/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
deleted file mode 100644
index da122da..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/PortProbe.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.servicemonitor;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.Socket;
-import java.util.Map;
-
-/**
- * Probe for a port being open.
- */
-public class PortProbe extends Probe {
-  protected static final Logger log = LoggerFactory.getLogger(PortProbe.class);
-  private final int port;
-  private final int timeout;
-
-  public PortProbe(int port, int timeout) {
-super("Port probe of " + port + " for " + timeout + "ms", null);
-this.port = port;
-this.timeout = timeout;
-  }
-
-  public static PortProbe create(Map props)
-  throws IOException {
-int port = getPropertyInt(props, PORT_PROBE_PORT, null);
-
-if (port >= 65536) {
-  throw new IOException(PORT_PROBE_PORT + " " + port + " is out of " +
-  "range");
-}
-
-int timeout = getPropertyInt(props, PORT_PROBE_CONNECT_TIMEOUT,
-PORT_PROBE_CONNECT_TIMEOUT_DEFAULT);
-
-return new PortProbe(port, timeout);
-  }
-
-  /**
-   * Try to connect to the (host,port); a failure to connect within
-   * the specified timeout is a failure.
-   * @param instance role instance
-   * @return the outcome
-   */
-  @Override
-  public ProbeStatus ping(ComponentInstance instance) {
-ProbeStatus status = new ProbeStatus();
-
-if (instance.getContainerStatus() == null || SliderUtils
-.isEmpty(instance.getContainerStatus().getIPs())) {
-  status.fail(this, new IOException(
-  instance.getCompInstanceName() + ": IP is not available yet"));
-  return status;
-}
-
-String ip = instance.getContainerStatus().getIPs().get(0);
-InetSocketAddress sockAddr = new InetSocketAddress(ip, port);
-Socket socket = new Socket();
-try {
-  if (log.isDebugEnabled()) {
-log.debug(instance.getCompInstanceName() + ": Connecting " + sockAddr
-.toString() + ", timeout=" + MonitorUtils
-.millisToHumanTime(timeout));
-  }
-  socket.connect(sockAddr, timeout);
-  status.succeed(this);
-} catch (Throwable e) {
-  String error =
-  instance.getCompInstanceName() + ": Probe " + sockAddr + " failed";
-  log.debug(error, e);
-  status.fail(this, new IOException(error, e));
-} finally {
-  IOUtils.closeSocket(socket);
-}
-return status;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/servicemonitor/Probe.java
 

[25/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
deleted file mode 100644
index 1fd49ab..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.launch;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import 
org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import 
org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.Serializable;
-import java.nio.ByteBuffer;
-import java.text.DateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static 
org.apache.hadoop.security.UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
-import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
-
-/**
- * Utils to work with credentials and tokens.
- *
- * Designed to be movable to Hadoop core
- */
-public final class CredentialUtils {
-
-  public static final String JOB_CREDENTIALS_BINARY
-  = SliderXmlConfKeys.MAPREDUCE_JOB_CREDENTIALS_BINARY;
-
-  private CredentialUtils() {
-  }
-
-  private static final Logger LOG =
-  LoggerFactory.getLogger(CredentialUtils.class);
-
-  /**
-   * Save credentials to a byte buffer. Returns null if there were no
-   * credentials to save
-   * @param credentials credential set
-   * @return a byte buffer of serialized tokens
-   * @throws IOException if the credentials could not be written to the stream
-   */
-  public static ByteBuffer marshallCredentials(Credentials credentials) throws 
IOException {
-ByteBuffer buffer = null;
-if (!credentials.getAllTokens().isEmpty()) {
-  DataOutputBuffer dob = new DataOutputBuffer();
-  try {
-credentials.writeTokenStorageToStream(dob);
-  } finally {
-dob.close();
-  }
-  buffer = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
-}
-return buffer;
-  }
-
-  public static File locateEnvCredentials(Map env,
-  Configuration conf,
-  StringBuffer sourceTextOut) throws FileNotFoundException {
-String tokenFilename = env.get(HADOOP_TOKEN_FILE_LOCATION);
-String source = "environment variable " + HADOOP_TOKEN_FILE_LOCATION;
-if (tokenFilename == null) {
-  tokenFilename = 

[53/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
deleted file mode 100644
index 83c9961..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderFactory.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.provider;
-
-import 
org.apache.hadoop.yarn.service.provider.defaultImpl.DefaultProviderFactory;
-import org.apache.hadoop.yarn.service.api.records.Artifact;
-import org.apache.hadoop.yarn.service.provider.docker.DockerProviderFactory;
-import org.apache.hadoop.yarn.service.provider.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class ProviderFactory {
-  protected static final Logger LOG =
-  LoggerFactory.getLogger(ProviderFactory.class);
-
-  protected ProviderFactory() {}
-
-  public abstract AbstractClientProvider createClientProvider();
-
-  public abstract ProviderService createServerProvider();
-
-  public static synchronized ProviderService getProviderService(Artifact
-  artifact) {
-return createSliderProviderFactory(artifact).createServerProvider();
-  }
-
-  public static synchronized AbstractClientProvider getClientProvider(Artifact
-  artifact) {
-return createSliderProviderFactory(artifact).createClientProvider();
-  }
-
-  /**
-   * Create a provider for a specific application
-   * @param artifact artifact
-   * @return provider factory
-   */
-  public static synchronized ProviderFactory createSliderProviderFactory(
-  Artifact artifact) {
-if (artifact == null || artifact.getType() == null) {
-  LOG.debug("Loading service provider type default");
-  return DefaultProviderFactory.getInstance();
-}
-LOG.debug("Loading service provider type {}", artifact.getType());
-switch (artifact.getType()) {
-  // TODO add handling for custom types?
-  // TODO handle application
-  case DOCKER:
-return DockerProviderFactory.getInstance();
-  case TARBALL:
-return TarballProviderFactory.getInstance();
-  default:
-throw new IllegalArgumentException(String.format("Resolution error, " +
-"%s should not be passed to createSliderProviderFactory",
-artifact.getType()));
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
deleted file mode 100644
index 9ef0176..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  

[18/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
deleted file mode 100644
index 2e40a9b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-public class SliderClusterProtocolProxy implements SliderClusterProtocol {
-
-  private static final RpcController NULL_CONTROLLER = null;
-  private final SliderClusterProtocolPB endpoint;
-  private final InetSocketAddress address;
-
-  public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint,
-  InetSocketAddress address) {
-Preconditions.checkArgument(endpoint != null, "null endpoint");
-Preconditions.checkNotNull(address != null, "null address");
-this.endpoint = endpoint;
-this.address = address;
-  }
-
-  @Override
-  public String toString() {
-final StringBuilder sb =
-new StringBuilder("SliderClusterProtocolProxy{");
-sb.append("address=").append(address);
-sb.append('}');
-return sb.toString();
-  }
-
-  @Override
-  public ProtocolSignature getProtocolSignature(String protocol,
-  long clientVersion,
-  int clientMethodsHash)
-  throws IOException {
-if (!protocol.equals(RPC.getProtocolName(SliderClusterProtocolPB.class))) {
-  throw new IOException("Serverside implements " +
-RPC.getProtocolName(SliderClusterProtocolPB.class) 
+
-". The following requested protocol is unknown: " +
-protocol);
-}
-
-return ProtocolSignature.getProtocolSignature(clientMethodsHash,
-RPC.getProtocolVersion(
-SliderClusterProtocol.class),
-SliderClusterProtocol.class);
-  }
-
-  @Override
-  public long getProtocolVersion(String protocol, long clientVersion)
-  throws IOException {
-return SliderClusterProtocol.versionID;
-  }
-  
-  private IOException convert(ServiceException se) {
-IOException ioe = ProtobufHelper.getRemoteException(se);
-if (ioe instanceof RemoteException) {
-  RemoteException remoteException = (RemoteException) ioe;
-  return remoteException.unwrapRemoteException();
-}
-return ioe;
-  }
-  
-  @Override public Messages.StopClusterResponseProto stopCluster(
-  Messages.StopClusterRequestProto request)
-  throws IOException, YarnException {
-try {
-  return endpoint.stopCluster(NULL_CONTROLLER, request);
-} catch (ServiceException e) {
-  throw convert(e);
-}
-  }
-
-  @Override
-  public Messages.UpgradeContainersResponseProto upgradeContainers(
-  Messages.UpgradeContainersRequestProto request) throws IOException,
-  YarnException {
-try {
-  return endpoint.upgradeContainers(NULL_CONTROLLER, request);
-} catch (ServiceException e) {
-  throw convert(e);
-}
-  }
-
-  

[29/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
deleted file mode 100644
index 51bde7b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
-commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
-  @Override
-  public String getActionName() {
-return SliderActions.ACTION_LIST;
-  }
-
-  @Parameter(names = {ARG_LIVE},
-  description = "List only live application instances")
-  public boolean live;
-
-  @Parameter(names = {ARG_STATE},
-  description = "list only applications in the specific YARN state")
-  public String state = "";
-  
-  @Parameter(names = {ARG_VERBOSE},
-  description = "print out information in details")
-  public boolean verbose = false;
-
-  @Parameter(names = {ARG_CONTAINERS},
-  description = "List containers of an application instance")
-  public boolean containers;
-
-  @Parameter(names = {ARG_VERSION},
-  description = "Filter containers by app version (used with " +
-ARG_CONTAINERS + ")")
-  public String version;
-
-  @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
-  description = "Filter containers by component names (used with " +
-ARG_CONTAINERS + ")")
-  public Set components = new HashSet<>(0);
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  public int getMinParams() {
-return 0;
-  }
-
-  @Override
-  public int getMaxParams() {
-return 1;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
deleted file mode 100644
index 012..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed 

[58/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
deleted file mode 100644
index e10305a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ConfigFormat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import java.util.Locale;
-
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public enum ConfigFormat {
-
-  JSON("json"),
-  PROPERTIES("properties"),
-  XML("xml"),
-  HADOOP_XML("hadoop_xml"),
-  ENV("env"),
-  TEMPLATE("template"),
-  YAML("yaml"),
-  ;
-  ConfigFormat(String suffix) {
-this.suffix = suffix;
-  }
-
-  private final String suffix;
-
-  public String getSuffix() {
-return suffix;
-  }
-
-
-  @Override
-  public String toString() {
-return suffix;
-  }
-
-  /**
-   * Get a matching format or null
-   * @param type
-   * @return the format
-   */
-  public static ConfigFormat resolve(String type) {
-for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
-return format;
-  }
-}
-return null;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
deleted file mode 100644
index 0ac508b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Configuration.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.api.records;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import 

[44/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/394183c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/394183c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/394183c7

Branch: refs/heads/yarn-native-services
Commit: 394183c796726aad70733a449a29c4571733b1d1
Parents: 4d5f448
Author: Billie Rinaldi 
Authored: Wed Aug 23 12:31:47 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:45:11 2017 -0700

--
 .../assemblies/hadoop-yarn-services-dist.xml|   30 +
 .../assemblies/hadoop-yarn-slider-dist.xml  |   30 -
 hadoop-yarn-project/hadoop-yarn/bin/yarn|8 +-
 .../service/webapp/ApplicationApiService.java   |  275 ++
 .../service/webapp/ApplicationApiWebApp.java|  123 +
 .../yarn/services/api/ApplicationApi.java   |   36 -
 .../api/impl/ApplicationApiService.java |  288 --
 .../services/webapp/ApplicationApiWebApp.java   |  125 -
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |3 -
 .../src/main/webapp/WEB-INF/web.xml |2 +-
 .../dev-support/findbugs-exclude.xml|   93 +-
 .../conf/slideram-log4j.properties  |   68 -
 .../conf/yarnservice-log4j.properties   |   62 +
 .../hadoop-yarn-slider-core/pom.xml |   30 +-
 .../slider/server/avro/RoleHistoryRecord.avsc   |  114 -
 .../apache/hadoop/security/KerberosDiags.java   |  677 -
 .../yarn/service/ContainerFailureTracker.java   |4 +-
 .../yarn/service/ContainerLaunchService.java|  101 -
 .../hadoop/yarn/service/ServiceContext.java |6 +-
 .../hadoop/yarn/service/ServiceMaster.java  |9 +-
 .../hadoop/yarn/service/ServiceMonitor.java |  149 -
 .../hadoop/yarn/service/ServiceScheduler.java   |   44 +-
 .../api/constants/ServiceApiConstants.java  |   69 +
 .../yarn/service/api/records/Application.java   |  466 +++
 .../service/api/records/ApplicationState.java   |   33 +
 .../service/api/records/ApplicationStatus.java  |  148 +
 .../yarn/service/api/records/Artifact.java  |  160 +
 .../yarn/service/api/records/BaseResource.java  |   52 +
 .../yarn/service/api/records/Component.java |  412 +++
 .../yarn/service/api/records/ConfigFile.java|  225 ++
 .../yarn/service/api/records/ConfigFormat.java  |   67 +
 .../yarn/service/api/records/Configuration.java |  225 ++
 .../yarn/service/api/records/Container.java |  297 ++
 .../service/api/records/ContainerState.java |   30 +
 .../hadoop/yarn/service/api/records/Error.java  |  129 +
 .../service/api/records/PlacementPolicy.java|  102 +
 .../service/api/records/ReadinessCheck.java |  175 ++
 .../yarn/service/api/records/Resource.java  |  159 +
 .../hadoop/yarn/service/client/ServiceCLI.java  |   14 +-
 .../yarn/service/client/ServiceClient.java  |  249 +-
 .../client/params/AbstractActionArgs.java   |   34 +-
 .../client/params/AbstractArgsDelegate.java |   28 +
 .../AbstractClusterBuildingActionArgs.java  |   58 +
 .../service/client/params/ActionBuildArgs.java  |2 -
 .../service/client/params/ActionClientArgs.java |   71 +
 .../service/client/params/ActionCreateArgs.java |2 -
 .../client/params/ActionDependencyArgs.java |4 +-
 .../service/client/params/ActionExistsArgs.java |   49 +
 .../service/client/params/ActionFlexArgs.java   |7 +-
 .../service/client/params/ActionFreezeArgs.java |   56 +
 .../service/client/params/ActionHelpArgs.java   |   44 +
 .../service/client/params/ActionKDiagArgs.java  |   76 +
 .../service/client/params/ActionKeytabArgs.java |   76 +
 .../service/client/params/ActionListArgs.java   |   76 +
 .../client/params/ActionRegistryArgs.java   |  218 ++
 .../client/params/ActionResolveArgs.java|  153 +
 .../client/params/ActionResourceArgs.java   |   70 +
 .../service/client/params/ActionStatusArgs.java |   51 +
 .../service/client/params/ActionThawArgs.java   |   67 +
 .../service/client/params/ActionTokensArgs.java |   78 +
 .../service/client/params/ActionUpdateArgs.java |   32 +
 .../yarn/service/client/params/ArgOps.java  |4 +-
 .../yarn/service/client/params/Arguments.java   |   37 +-
 .../yarn/service/client/params/ClientArgs.java  |  147 +-
 .../yarn/service/client/params/CommonArgs.java  |   30 +-
 .../client/params/ComponentArgsDelegate.java|4 +-
 .../client/params/DontSplitArguments.java   |   34 +
 .../client/params/LaunchArgsAccessor.java   |   30 +
 .../client/params/LaunchArgsDelegate.java   |   51 +
 .../client/params/OptionArgsDelegate.java   |   66 +
 .../client/params/PathArgumentConverter.java|   34 +
 .../client/params/SliderAMCreateAction.java |4 -
 .../service/client/params/SliderActions.java|   48 +-
 

[24/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
deleted file mode 100644
index 1964459..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/PersistKeys.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-public class PersistKeys {
-
-  public static final String SCHEMA =
-"http://example.org/specification/v2.0.0;;
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
deleted file mode 100644
index ac8fca5..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.registry;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-
-/**
- * Miscellaneous methods to assist slider registry work
- * 
- */
-public class SliderRegistryUtils {
-
-
-  /**
-   * Get the registry path for an instance under the user's home node
-   * @param instanceName application instance
-   * @return a path to the registry location for this application instance.
-   */
-  public static String registryPathForInstance(String instanceName) {
-return RegistryUtils.servicePath(
-RegistryUtils.currentUser(), SliderKeys.APP_TYPE, instanceName
-);
-  }
-
-  /**
-   * Process a path expanding it if needed.
-   * Validation is delegated to later as the core registry will need
-   * to do that anyway
-   * @param path path
-   * @return a path maybe with some expansion
-   */
-  public static String resolvePath(String path) {
-Preconditions.checkArgument(path!=null, "null path");
-Preconditions.checkArgument(!path.isEmpty(), "empty path");
-String newpath = path;
-if (path.startsWith("~/")) {
-  // add user expansion
-  newpath = RegistryUtils.homePathForCurrentUser() + path.substring(1);
-} else if (path.equals("~")) {
-  newpath = 

[62/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
new file mode 100644
index 000..6e6f4dd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/SliderUtils.java
@@ -0,0 +1,1654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.GlobFilter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
+import org.apache.hadoop.yarn.service.exceptions.BadCommandArgumentsException;
+import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.LauncherExitCodes;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.zookeeper.server.util.KerberosUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.URL;
+import java.net.URLDecoder;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPOutputStream;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+/**
+ * These are slider-specific Util methods
+ */
+public final class SliderUtils {
+
+  private static final Logger log = LoggerFactory.getLogger(SliderUtils.class);
+
+  /**

[07/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
deleted file mode 100644
index 395ff22..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.java
+++ /dev/null
@@ -1,380 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.appstate;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.slider.api.types.NodeInformation;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.providers.PlacementPolicy;
-import org.apache.slider.server.appmaster.model.mock.MockAppState;
-import org.apache.slider.server.appmaster.model.mock.MockFactory;
-import org.apache.slider.server.appmaster.model.mock.MockRoles;
-import org.apache.slider.server.appmaster.model.mock.MockYarnEngine;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.state.AppState;
-import org.apache.slider.server.appmaster.state.AppState.NodeUpdatedOutcome;
-import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
-import org.apache.slider.server.appmaster.state.ContainerAssignment;
-import org.apache.slider.server.appmaster.state.NodeInstance;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
-import static 
org.apache.slider.server.appmaster.model.mock.MockFactory.AAROLE_2;
-
-/**
- * Test Anti-affine placement.
- */
-public class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
-implements MockRoles {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(TestMockAppStateAAPlacement.class);
-
-  private static final int NODES = 3;
-
-  /**
-   * The YARN engine has a cluster with very few nodes (3) and lots of
-   * containers, so if AA placement isn't working, there will be affine
-   * placements surfacing.
-   * @return
-   */
-  @Override
-  public MockYarnEngine createYarnEngine() {
-return new MockYarnEngine(NODES, 8);
-  }
-
-  /**
-   * This is the simplest AA allocation: no labels, so allocate anywhere.
-   * @throws Throwable
-   */
-  //@Test
-  public void testAllocateAANoLabel() throws Throwable {
-RoleStatus aaRole = getAaRole();
-
-assertTrue(cloneNodemap().size() > 0);
-
-// want multiple instances, so there will be iterations
-aaRole.setDesired(2);
-
-List ops = appState.reviewRequestAndReleaseNodes();
-AMRMClient.ContainerRequest request = getSingleRequest(ops);
-assertFalse(request.getRelaxLocality());
-assertEquals(request.getNodes().size(), engine.getCluster()
-.getClusterSize());
-assertNull(request.getRacks());
-assertNotNull(request.getCapability());
-
-Container allocated = engine.allocateContainer(request);
-
-// notify the container ane expect
-List assignments = new ArrayList<>();
-List operations = new ArrayList<>();
-

[21/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
deleted file mode 100644
index 06dde67..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ /dev/null
@@ -1,2138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you under the Apache License, Version 2.0 (the
- *  "License"); you may not use this file except in compliance
- *  with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import com.codahale.metrics.MetricRegistry;
-import com.codahale.metrics.health.HealthCheckRegistry;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import 
org.apache.hadoop.registry.server.integration.RMRegistryOperationsService;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.service.ServiceOperations;
-import org.apache.hadoop.service.ServiceStateChangeListener;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.NodeState;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.client.api.TimelineV2Client;
-import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
-import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import 
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
-import 

[33/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
deleted file mode 100644
index e89306c..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Set of configuration properties that can be injected into the application
- * components via envs, files and custom pluggable helper docker containers.
- * Files of several standard formats like xml, properties, json, yaml and
- * templates will be supported.
- **/
-
-@ApiModel(description = "Set of configuration properties that can be injected 
into the application components via envs, files and custom pluggable helper 
docker containers. Files of several standard formats like xml, properties, 
json, yaml and templates will be supported.")
-@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration implements Serializable {
-  private static final long serialVersionUID = -4330788704981074466L;
-
-  private Map properties = new HashMap();
-  private Map env = new HashMap();
-  private List files = new ArrayList();
-
-  /**
-   * A blob of key-value pairs of common application properties.
-   **/
-  public Configuration properties(Map properties) {
-this.properties = properties;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "A blob of key-value pairs of 
common application properties.")
-  @JsonProperty("properties")
-  public Map getProperties() {
-return properties;
-  }
-
-  public void setProperties(Map properties) {
-this.properties = properties;
-  }
-
-  /**
-   * A blob of key-value pairs which will be appended to the default system
-   * properties and handed off to the application at start time. All 
placeholder
-   * references to properties will be substituted before injection.
-   **/
-  public Configuration env(Map env) {
-this.env = env;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "A blob of key-value pairs which 
will be appended to the default system properties and handed off to the 
application at start time. All placeholder references to properties will be 
substituted before injection.")
-  @JsonProperty("env")
-  public Map getEnv() {
-return env;
-  }
-
-  public void setEnv(Map env) {
-this.env = env;
-  }
-
-  /**
-   * Array of list of files that needs to be created and made available as
-   * volumes in the application component containers.
-   **/
-  public Configuration files(List files) {
-this.files = files;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "Array of list of files that 
needs to be created and made available as volumes in the application component 
containers.")
-  

[10/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
deleted file mode 100644
index da06c13..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/resources/org/apache/slider/providers/agent/conf/command_template.json
+++ /dev/null
@@ -1,168 +0,0 @@
-{
-  "roleCommand": "{{COMMAND}}",
-  "clusterName": "{{CLUSTER_NAME}}",
-  "hostname": "{{HOST_NAME}}",
-  "hostLevelParams": {
-"java_home": "/usr/jdk64/jdk1.7.0_45"
-  },
-  "commandType": "EXECUTION_COMMAND",
-  "roleParams": {},
-  "serviceName": "{{SERVICE_NAME}}",
-  "role": "{{ROLE_NAME}}",
-  "commandParams": {},
-  "taskId": "{{TASK_ID}}",
-  "public_hostname": "{{HOST_NAME}}",
-  "configurations": {
-"hbase-log4j": {
-  "log4j.threshold": "ALL",
-  "log4j.rootLogger": "${hbase.root.logger}",
-  "log4j.logger.org.apache.zookeeper": "INFO",
-  "log4j.logger.org.apache.hadoop.hbase": "DEBUG",
-  "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher": 
"INFO",
-  "log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil": "INFO",
-  "log4j.category.SecurityLogger": "${hbase.security.logger}",
-  "log4j.appender.console": "org.apache.log4j.ConsoleAppender",
-  "log4j.appender.console.target": "System.err",
-  "log4j.appender.console.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.console.layout.ConversionPattern": "%d{ISO8601} %-5p 
[%t] %c{2}: %m%n",
-  "log4j.appender.RFAS": "org.apache.log4j.RollingFileAppender",
-  "log4j.appender.RFAS.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.RFAS.layout.ConversionPattern": "%d{ISO8601} %p %c: 
%m%n",
-  "log4j.appender.RFAS.MaxFileSize": "${hbase.security.log.maxfilesize}",
-  "log4j.appender.RFAS.MaxBackupIndex": 
"${hbase.security.log.maxbackupindex}",
-  "log4j.appender.RFAS.File": 
"${hbase.log.dir}/${hbase.security.log.file}",
-  "log4j.appender.RFA": "org.apache.log4j.RollingFileAppender",
-  "log4j.appender.RFA.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.RFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] 
%c{2}: %m%n",
-  "log4j.appender.RFA.MaxFileSize": "${hbase.log.maxfilesize}",
-  "log4j.appender.RFA.MaxBackupIndex": "${hbase.log.maxbackupindex}",
-  "log4j.appender.RFA.File": "${hbase.log.dir}/${hbase.log.file}",
-  "log4j.appender.NullAppender": "org.apache.log4j.varia.NullAppender",
-  "log4j.appender.DRFA": "org.apache.log4j.DailyRollingFileAppender",
-  "log4j.appender.DRFA.layout": "org.apache.log4j.PatternLayout",
-  "log4j.appender.DRFA.layout.ConversionPattern": "%d{ISO8601} %-5p [%t] 
%c{2}: %m%n",
-  "log4j.appender.DRFA.File": "${hbase.log.dir}/${hbase.log.file}",
-  "log4j.appender.DRFA.DatePattern": ".-MM-dd",
-  "log4j.additivity.SecurityLogger": "false",
-  "hbase.security.logger": "INFO,console",
-  "hbase.security.log.maxfilesize": "256MB",
-  "hbase.security.log.maxbackupindex": "20",
-  "hbase.security.log.file": "SecurityAuth.audit",
-  "hbase.root.logger": "INFO,console",
-  "hbase.log.maxfilesize": "256MB",
-  "hbase.log.maxbackupindex": "20",
-  "hbase.log.file": "hbase.log",
-  "hbase.log.dir": "."
-},
-"global": {
-  "hbase_root": "{{HBASE_HOME}}",
-  "hbase_pid_dir": "{{PID_DIR}}",
-  "proxyuser_group": "users",
-  "syncLimit": "5",
-  "hbase_regionserver_heapsize": "{{REGION_SERVER_HEAP_SIZE}}",
-  "rca_enabled": "false",
-  "tickTime": "2000",
-  "hbase_master_heapsize": "{{MASTER_HEAP_SIZE}}",
-  "initLimit": "10",
-  "user_group": "{{GROUP_NAME}}",
-  "hbase_user": "{{USER_NAME}}",
-  "hbase_log_dir": "{{LOG_DIR}}"
-},
-"hdfs-site": {
-  "dfs.namenode.checkpoint.period": "21600",
-  "dfs.namenode.avoid.write.stale.datanode": "true",
-  "dfs.namenode.checkpoint.txns": "100",
-  "dfs.block.access.token.enable": "true",
-  "dfs.support.append": "true",
-  "dfs.datanode.address": "0.0.0.0:${ambari.dfs.datanode.port}",
-  "dfs.cluster.administrators": " hdfs",
-  "dfs.replication": "3",
-  "ambari.dfs.datanode.http.port": "50075",
-  "dfs.datanode.balance.bandwidthPerSec": "625",
-  "dfs.namenode.safemode.threshold-pct": "1.0f",
-  

[16/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
deleted file mode 100644
index 3e8a3c3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAssignment.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Container;
-
-/**
- * Static assignment structure
- */
-public class ContainerAssignment {
-
-  /**
-   * Container that has been allocated
-   */
-  public final Container container;
-
-  /**
-   * Role to assign to it
-   */
-  public final RoleStatus role;
-
-  /**
-   * Placement outcome: was this from history or not
-   */
-  public final ContainerAllocationOutcome placement;
-
-  public ContainerAssignment(Container container,
-  RoleStatus role,
-  ContainerAllocationOutcome placement) {
-this.container = container;
-this.role = role;
-this.placement = placement;
-  }
-
-  @Override
-  public String toString() {
-final StringBuilder sb = new StringBuilder("ContainerAssignment{");
-sb.append("container=").append(container);
-sb.append(", role=").append(role);
-sb.append(", placement=").append(placement);
-sb.append('}');
-return sb.toString();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
deleted file mode 100644
index 6df4bf4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
-
-/**
- * Container outcomes we care about; slightly simplified from
- * {@link ContainerExitStatus} -and hopefully able to handle
- * any new exit codes.
- */
-public enum ContainerOutcome {
-  Completed,
-  Failed,
-  Failed_limits_exceeded,
-  Disk_failure,
-  Preempted;
-
-  /**
-   * Build a container outcome from 

[52/75] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db5888ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
deleted file mode 100644
index b3fdef9..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/rest/UgiJerseyBinding.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.rest;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.config.ClientConfig;
-import com.sun.jersey.api.client.config.DefaultClientConfig;
-import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
-import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
-import org.apache.hadoop.conf.Configuration;
-import 
org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-
-/**
- * Class to bond to a Jersey client, for UGI integration and SPNEGO.
- * 
- *   Usage: create an instance, then when creating a Jersey Client
- *   pass in to the constructor the handler provided by {@link #getHandler()}
- *
- * see https://jersey.java.net/apidocs/1.17/jersey/com/sun/jersey/client/urlconnection/HttpURLConnectionFactory.html;>Jersey
 docs
- */
-public class UgiJerseyBinding implements
-HttpURLConnectionFactory {
-  private static final Logger log =
-  LoggerFactory.getLogger(UgiJerseyBinding.class);
-
-  private final UrlConnectionOperations operations;
-  private final URLConnectionClientHandler handler;
-
-  /**
-   * Construct an instance
-   * @param operations operations instance
-   */
-  @SuppressWarnings("ThisEscapedInObjectConstruction")
-  public UgiJerseyBinding(UrlConnectionOperations operations) {
-Preconditions.checkArgument(operations != null, "Null operations");
-this.operations = operations;
-handler = new URLConnectionClientHandler(this);
-  }
-
-  /**
-   * Create an instance off the configuration. The SPNEGO policy
-   * is derived from the current UGI settings.
-   * @param conf config
-   */
-  public UgiJerseyBinding(Configuration conf) {
-this(new UrlConnectionOperations(conf));
-  }
-
-  /**
-   * Get a URL connection. 
-   * @param url URL to connect to
-   * @return the connection
-   * @throws IOException any problem. {@link AuthenticationException} 
-   * errors are wrapped
-   */
-  @Override
-  public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
-try {
-  // open a connection handling status codes and so redirections
-  // but as it opens a connection, it's less useful than you think.
-
-  return operations.openConnection(url);
-} catch (AuthenticationException e) {
-  throw new IOException(e);
-}
-  }
-
-  public UrlConnectionOperations getOperations() {
-return operations;
-  }
-
-  public URLConnectionClientHandler getHandler() {
-return handler;
-  }
-  
-  /**
-   * Get the SPNEGO flag (as found in the operations instance
-   * @return the spnego policy
-   */
-  public boolean isUseSpnego() {
-return operations.isUseSpnego();
-  }
-
-
-  /**
-   * Uprate error codes 400 and up into faults; 
-   * 
-   * see {@link ExceptionConverter#convertJerseyException(String, String, 
UniformInterfaceException)}
-   */
-  public static IOException uprateFaults(HttpVerb verb, String url,
- 

[26/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
deleted file mode 100644
index 86025ee..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderVersionInfo.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import org.apache.hadoop.util.VersionInfo;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.Locale;
-import java.util.Properties;
-
-/**
- * Extract the version properties, which will look something like
- * 
- * application.name=${pom.name}
- * application.version=${pom.version}
- * application.build=${buildNumber}
- * application.build.java.version=${java.version}
- * application.build.info=${pom.name}-${pom.version} Built against 
${buildNumber} on ${java.version} by ${user.name}
- * 
- * 
- * the mvn process-resources target will expand the properties
- * and add the resources to target/classes, which will then look something like
- * 
- *   application.name=Slider Core
- *   application.version=0.7.1-SNAPSHOT
- *   application.build=1dd69
- *   application.build.java.version=1.7.0_45
- *   application.build.user=stevel
- *   application.build.info=Slider Core-0.7.1-SNAPSHOT Built against 1dd69 on 
1.7.0_45 by stevel
- * 
- * 
- * Note: the values will change and more properties added.
- */
-public class SliderVersionInfo {
-  private static final Logger log = 
LoggerFactory.getLogger(SliderVersionInfo.class);
-
-  /**
-   * Name of the resource containing the filled-in-at-runtime props
-   */
-  public static final String VERSION_RESOURCE =
-  "org/apache/slider/providers/dynamic/application.properties";
-
-  public static final String APP_NAME = "application.name";
-  public static final String APP_VERSION = "application.version";
-  public static final String APP_BUILD = "application.build";
-  public static final String APP_BUILD_JAVA_VERSION = 
"application.build.java.version";
-  public static final String APP_BUILD_USER = "application.build.user";
-  public static final String APP_BUILD_INFO = "application.build.info";
-  public static final String HADOOP_BUILD_INFO = "hadoop.build.info";
-  public static final String HADOOP_DEPLOYED_INFO = "hadoop.deployed.info";
-
-
-  public static Properties loadVersionProperties()  {
-Properties props = new Properties();
-URL resURL = SliderVersionInfo.class.getClassLoader()
-   .getResource(VERSION_RESOURCE);
-assert resURL != null : "Null resource " + VERSION_RESOURCE;
-
-try {
-  InputStream inStream = resURL.openStream();
-  assert inStream != null : "Null input stream from " + VERSION_RESOURCE;
-  props.load(inStream);
-} catch (IOException e) {
-  log.warn("IOE loading " + VERSION_RESOURCE, e);
-}
-return props;
-  }
-
-  /**
-   * Load the version info and print it
-   * @param logger logger
-   */
-  public static void loadAndPrintVersionInfo(Logger logger) {
-Properties props = loadVersionProperties();
-logger.info(props.getProperty(APP_BUILD_INFO));
-logger.info("Compiled against Hadoop {}",
-props.getProperty(HADOOP_BUILD_INFO));
-logger.info(getHadoopVersionString());
-  }
-  
-  public static String getHadoopVersionString() {
-return String.format(Locale.ENGLISH,
-"Hadoop runtime version %s with source checksum %s and build date %s",
-VersionInfo.getBranch(),
-

[01/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He [Forced Update!]

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services 90a6b0993 -> 6e7ddba0c (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
deleted file mode 100644
index 5493198..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/utils/TestUtility.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.utils;
-
-import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
-import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
-import org.apache.commons.compress.utils.IOUtils;
-import org.junit.Assert;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-
-/**
- *  Various utility methods
- *  Byte comparison methods are from
- *  org.apache.hadoop.fs.contract.ContractTestUtils
- */
-public class TestUtility {
-  protected static final Logger log =
-  LoggerFactory.getLogger(TestUtility.class);
-
-  public static void addDir(File dirObj, ZipArchiveOutputStream zipFile, 
String prefix) throws IOException {
-for (File file : dirObj.listFiles()) {
-  if (file.isDirectory()) {
-addDir(file, zipFile, prefix + file.getName() + File.separator);
-  } else {
-log.info("Adding to zip - " + prefix + file.getName());
-zipFile.putArchiveEntry(new ZipArchiveEntry(prefix + file.getName()));
-IOUtils.copy(new FileInputStream(file), zipFile);
-zipFile.closeArchiveEntry();
-  }
-}
-  }
-
-  public static void zipDir(String zipFile, String dir) throws IOException {
-File dirObj = new File(dir);
-ZipArchiveOutputStream out = new ZipArchiveOutputStream(new 
FileOutputStream(zipFile));
-log.info("Creating : {}", zipFile);
-try {
-  addDir(dirObj, out, "");
-} finally {
-  out.close();
-}
-  }
-
-  public static String createAppPackage(
-  TemporaryFolder folder, String subDir, String pkgName, String srcPath) 
throws IOException {
-String zipFileName;
-File pkgPath = folder.newFolder(subDir);
-File zipFile = new File(pkgPath, pkgName).getAbsoluteFile();
-zipFileName = zipFile.getAbsolutePath();
-TestUtility.zipDir(zipFileName, srcPath);
-log.info("Created temporary zip file at {}", zipFileName);
-return zipFileName;
-  }
-
-
-  /**
-   * Assert that tthe array original[0..len] and received[] are equal.
-   * A failure triggers the logging of the bytes near where the first
-   * difference surfaces.
-   * @param original source data
-   * @param received actual
-   * @param len length of bytes to compare
-   */
-  public static void compareByteArrays(byte[] original,
-  byte[] received,
-  int len) {
-Assert.assertEquals("Number of bytes read != number written",
-len, received.length);
-int errors = 0;
-int first_error_byte = -1;
-for (int i = 0; i < len; i++) {
-  if (original[i] != received[i]) {
-if (errors == 0) {
-  first_error_byte = i;
-}
-errors++;
-  }
-}
-
-if (errors > 0) {
-  String message = String.format(" %d errors in file of length %d",
-  errors, len);
-  log.warn(message);
-  // the range either side of the first error to print
-  // this is a purely arbitrary number, to aid user debugging
-  final int overlap = 10;
-  for (int i = Math.max(0, first_error_byte - overlap);
-   i < 

[42/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
new file mode 100644
index 000..b57225a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ApplicationStatus.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * The current status of a submitted application, returned as a response to the
+ * GET API.
+ **/
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@ApiModel(description = "The current status of a submitted application, 
returned as a response to the GET API.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class ApplicationStatus extends BaseResource {
+  private static final long serialVersionUID = -3469885905347851034L;
+
+  private String diagnostics = null;
+  private ApplicationState state = null;
+  private Integer code = null;
+
+  /**
+   * Diagnostic information (if any) for the reason of the current state of the
+   * application. It typically has a non-null value, if the application is in a
+   * non-running state.
+   **/
+  public ApplicationStatus diagnostics(String diagnostics) {
+this.diagnostics = diagnostics;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Diagnostic information (if any) 
for the reason of the current state of the application. It typically has a 
non-null value, if the application is in a non-running state.")
+  @JsonProperty("diagnostics")
+  public String getDiagnostics() {
+return diagnostics;
+  }
+
+  public void setDiagnostics(String diagnostics) {
+this.diagnostics = diagnostics;
+  }
+
+  /**
+   * Application state.
+   **/
+  public ApplicationStatus state(ApplicationState state) {
+this.state = state;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Application state.")
+  @JsonProperty("state")
+  public ApplicationState getState() {
+return state;
+  }
+
+  public void setState(ApplicationState state) {
+this.state = state;
+  }
+
+  /**
+   * An error code specific to a scenario which app owners should be able to 
use
+   * to understand the failure in addition to the diagnostic information.
+   **/
+  public ApplicationStatus code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "An error code specific to a 
scenario which app owners should be able to use to understand the failure in 
addition to the diagnostic information.")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+ApplicationStatus applicationStatus = (ApplicationStatus) o;
+return Objects.equals(this.diagnostics, 

[37/75] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
new file mode 100644
index 000..b851fb7
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/Probe.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.servicemonitor.probe;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.service.compinstance.ComponentInstance;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Base class of all probes.
+ */
+public abstract class Probe implements MonitorKeys {
+
+  protected final Configuration conf;
+  private String name;
+
+  /**
+   * Create a probe of a specific name
+   *
+   * @param name probe name
+   * @param conf configuration being stored.
+   */
+  public Probe(String name, Configuration conf) {
+this.name = name;
+this.conf = conf;
+  }
+
+
+  protected void setName(String name) {
+this.name = name;
+  }
+
+  public String getName() {
+return name;
+  }
+
+
+  @Override
+  public String toString() {
+return getName();
+  }
+
+  public static String getProperty(Map props, String name,
+  String defaultValue) throws IOException {
+String value = props.get(name);
+if (StringUtils.isEmpty(value)) {
+  if (defaultValue == null) {
+throw new IOException(name + " not specified");
+  }
+  return defaultValue;
+}
+return value;
+  }
+
+  public static int getPropertyInt(Map props, String name,
+  Integer defaultValue) throws IOException {
+String value = props.get(name);
+if (StringUtils.isEmpty(value)) {
+  if (defaultValue == null) {
+throw new IOException(name + " not specified");
+  }
+  return defaultValue;
+}
+return Integer.parseInt(value);
+  }
+
+  /**
+   * perform any prelaunch initialization
+   */
+  public void init() throws IOException {
+
+  }
+
+  /**
+   * Ping the endpoint. All exceptions must be caught and included in the
+   * (failure) status.
+   *
+   * @param instance instance to ping
+   * @return the status
+   */
+  public abstract ProbeStatus ping(ComponentInstance instance);
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/394183c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
new file mode 100644
index 000..7cd761c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/servicemonitor/probe/ProbeStatus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache 

hadoop git commit: YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by Eric Yang

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services 54ad59a47 -> 90a6b0993


YARN-7100. Added JsonIgnore annotation to Resource#getMemoryMB. Contributed by 
Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90a6b099
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90a6b099
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90a6b099

Branch: refs/heads/yarn-native-services
Commit: 90a6b099379b5b86da338564e5b0f72a6bdaa69c
Parents: 54ad59a
Author: Jian He 
Authored: Tue Aug 29 20:37:38 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:37:38 2017 -0700

--
 .../java/org/apache/hadoop/yarn/service/api/records/Resource.java  | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/90a6b099/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
index cec9de9..dfdf92a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Resource.java
@@ -22,6 +22,7 @@ import io.swagger.annotations.ApiModelProperty;
 
 import java.util.Objects;
 
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -103,6 +104,7 @@ public class Resource extends BaseResource implements 
Cloneable {
 this.memory = memory;
   }
 
+  @JsonIgnore
   public long getMemoryMB() {
 if (this.memory == null) {
   return 0;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6756. ContainerRequest#executionTypeRequest causes NPE. Contributed by Jian He

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 9bcd64d23 -> 036cbd745


YARN-6756. ContainerRequest#executionTypeRequest causes NPE. Contributed by 
Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/036cbd74
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/036cbd74
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/036cbd74

Branch: refs/heads/branch-2
Commit: 036cbd745313598a8867d7fff11d110c90c2961a
Parents: 9bcd64d
Author: Jian He 
Authored: Tue Aug 29 20:35:53 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:36:23 2017 -0700

--
 .../java/org/apache/hadoop/yarn/client/api/AMRMClient.java | 5 +++--
 .../yarn/client/api/impl/TestAMRMClientContainerRequest.java   | 6 --
 2 files changed, 7 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/036cbd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index bc4173f..1c3ac1c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -111,9 +111,10 @@ public abstract class AMRMClient extends
 private List racks;
 private Priority priority;
 private long allocationRequestId;
-private boolean relaxLocality;
+private boolean relaxLocality = true;
 private String nodeLabelsExpression;
-private ExecutionTypeRequest executionTypeRequest;
+private ExecutionTypeRequest executionTypeRequest =
+ExecutionTypeRequest.newInstance();
 
 /**
  * Instantiates a {@link ContainerRequest} with the given constraints and

http://git-wip-us.apache.org/repos/asf/hadoop/blob/036cbd74/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
index ad18da3..9603539 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
@@ -51,8 +51,10 @@ public class TestAMRMClientContainerRequest {
 
 Resource capability = Resource.newInstance(1024, 1);
 ContainerRequest request =
-new ContainerRequest(capability, new String[] {"host1", "host2"},
-new String[] {"/rack2"}, Priority.newInstance(1));
+ContainerRequest.newBuilder().capability(capability)
+.nodes(new String[] { "host1", "host2" })
+.racks(new String[] { "/rack2" }).priority(Priority.newInstance(1))
+.build();
 client.addContainerRequest(request);
 verifyResourceRequest(client, request, "host1", true);
 verifyResourceRequest(client, request, "host2", true);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6756. ContainerRequest#executionTypeRequest causes NPE. Contributed by Jian He

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk cf93d60d3 -> 8201ed800


YARN-6756. ContainerRequest#executionTypeRequest causes NPE. Contributed by 
Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8201ed80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8201ed80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8201ed80

Branch: refs/heads/trunk
Commit: 8201ed8009e5f04c49568a8133635d47fcde3989
Parents: cf93d60
Author: Jian He 
Authored: Tue Aug 29 20:35:53 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 20:35:53 2017 -0700

--
 .../java/org/apache/hadoop/yarn/client/api/AMRMClient.java | 5 +++--
 .../yarn/client/api/impl/TestAMRMClientContainerRequest.java   | 6 --
 2 files changed, 7 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8201ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 69f3777..60e305f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -122,9 +122,10 @@ public abstract class AMRMClient extends
 private List racks;
 private Priority priority;
 private long allocationRequestId;
-private boolean relaxLocality;
+private boolean relaxLocality = true;
 private String nodeLabelsExpression;
-private ExecutionTypeRequest executionTypeRequest;
+private ExecutionTypeRequest executionTypeRequest =
+ExecutionTypeRequest.newInstance();
 
 /**
  * Instantiates a {@link ContainerRequest} with the given constraints and

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8201ed80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
index ad18da3..9603539 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
@@ -51,8 +51,10 @@ public class TestAMRMClientContainerRequest {
 
 Resource capability = Resource.newInstance(1024, 1);
 ContainerRequest request =
-new ContainerRequest(capability, new String[] {"host1", "host2"},
-new String[] {"/rack2"}, Priority.newInstance(1));
+ContainerRequest.newBuilder().capability(capability)
+.nodes(new String[] { "host1", "host2" })
+.racks(new String[] { "/rack2" }).priority(Priority.newInstance(1))
+.build();
 client.addContainerRequest(request);
 verifyResourceRequest(client, request, "host1", true);
 verifyResourceRequest(client, request, "host2", true);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

svn commit: r1806657 - /hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml

[xiao]

Author: xiao
Date: Wed Aug 30 01:30:38 2017
New Revision: 1806657

URL: http://svn.apache.org/viewvc?rev=1806657=rev
Log:
Add xiao to PMC list

Modified:
hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml

Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml
URL: 
http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1806657=1806656=1806657=diff
==
--- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml 
(original)
+++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Wed 
Aug 30 01:30:38 2017
@@ -583,6 +583,14 @@
 -8
 
 
+   
+ xiao
+ Xiao Chen
+ Cloudera
+ HDFS
+ -8
+   
+
   
 xyao
 http://people.apache.org/~xyao;>Xiaoyu Yao



-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-12191. Provide option to not capture the accessTime change of a file to snapshot if no other modification has been done to this file. Contributed by Yongjun Zhang.

[yjzhangal]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 26fafc359 -> cf93d60d3


HDFS-12191. Provide option to not capture the accessTime change of a file to 
snapshot if no other modification has been done to this file. Contributed by 
Yongjun Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf93d60d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf93d60d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf93d60d

Branch: refs/heads/trunk
Commit: cf93d60d3f032000e5b78a08d320793d78799f3d
Parents: 26fafc3
Author: Yongjun Zhang 
Authored: Tue Aug 29 16:57:03 2017 -0700
Committer: Yongjun Zhang 
Committed: Tue Aug 29 16:57:03 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   3 +
 .../hdfs/server/namenode/FSDirAttrOp.java   |   4 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   4 +-
 .../hadoop/hdfs/server/namenode/INode.java  |   7 +-
 .../namenode/snapshot/SnapshotManager.java  |  28 
 .../src/main/resources/hdfs-default.xml |  44 +++--
 .../hdfs/server/namenode/TestFSDirAttrOp.java   |   7 +
 .../snapshot/TestSnapshotDiffReport.java| 163 ++-
 8 files changed, 233 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf93d60d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b836ff8..17cabad 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -357,6 +357,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
   HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT;
 
+  public static final String 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE = 
"dfs.namenode.snapshot.skip.capture.accesstime-only-change";
+  public static final boolean 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE_DEFAULT = false;
+
   // Whether to enable datanode's stale state detection and usage for reads
   public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = 
"dfs.namenode.avoid.read.stale.datanode";
   public static final boolean 
DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf93d60d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index d4b24f5..0dfaa8e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -494,7 +494,9 @@ public class FSDirAttrOp {
 // then no need to store access time
 if (atime != -1 && (status || force
 || atime > inode.getAccessTime() + fsd.getAccessTimePrecision())) {
-  inode.setAccessTime(atime, latest);
+  inode.setAccessTime(atime, latest,
+  fsd.getFSNamesystem().getSnapshotManager().
+  getSkipCaptureAccessTimeOnlyChange());
   status = true;
 }
 return status;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf93d60d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 060bd59..bc62a7e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -413,7 +413,7 @@ public class FSEditLogLoader {
   // update the block list.
   
   // Update the salient file attributes.
-  newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
+  

hadoop git commit: YARN-7077. TestAMSimulator and TestNMSimulator fail. (Contributed by Akira Ajisaka via Yufei Gu)

[yufei]

Repository: hadoop
Updated Branches:
  refs/heads/trunk f59332b97 -> 26fafc359


YARN-7077. TestAMSimulator and TestNMSimulator fail. (Contributed by Akira 
Ajisaka via Yufei Gu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26fafc35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26fafc35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26fafc35

Branch: refs/heads/trunk
Commit: 26fafc359787eae0ef82196000f4a04956b2abaa
Parents: f59332b
Author: Yufei Gu 
Authored: Tue Aug 29 16:33:29 2017 -0700
Committer: Yufei Gu 
Committed: Tue Aug 29 16:33:51 2017 -0700

--
 .../main/java/org/apache/hadoop/yarn/sls/SLSRunner.java   |  4 
 hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml  | 10 --
 2 files changed, 4 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/26fafc35/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
index 477cc4a..59f9c17 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/SLSRunner.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
 import 
org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import 
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
@@ -227,6 +228,9 @@ public class SLSRunner extends Configured implements Tool {
 if (Class.forName(schedulerClass) == CapacityScheduler.class) {
   rmConf.set(YarnConfiguration.RM_SCHEDULER,
   SLSCapacityScheduler.class.getName());
+  rmConf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+  rmConf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+  ProportionalCapacityPreemptionPolicy.class.getName());
 } else if (Class.forName(schedulerClass) == FairScheduler.class) {
   rmConf.set(YarnConfiguration.RM_SCHEDULER,
   SLSFairScheduler.class.getName());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/26fafc35/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml
--
diff --git a/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml 
b/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml
index 7b2e674..20146fc 100644
--- a/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml
+++ b/hadoop-tools/hadoop-sls/src/test/resources/yarn-site.xml
@@ -22,16 +22,6 @@
   
 
   
-yarn.resourcemanager.scheduler.monitor.enable
-true
-  
-
-  
-yarn.resourcemanager.scheduler.monitor.policies
-
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy
-  
-
-  
 The address of the RM web application.
 yarn.resourcemanager.webapp.address
 localhost:18088


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-5219. When an export var command fails in launch_container.sh, the full container launch should fail. (Sunil G via wangda)

[wangda]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 b906bddd9 -> 9bcd64d23


YARN-5219. When an export var command fails in launch_container.sh, the full 
container launch should fail. (Sunil G via wangda)

Change-Id: Iaa6b978bb89482e9d1d77ba57f4adfdc48e39a3c
(cherry picked from commit f59332b97b9a57e3cf1dcdeb47d7838d287100eb)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9bcd64d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9bcd64d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9bcd64d2

Branch: refs/heads/branch-2
Commit: 9bcd64d233a9f93b2ef8111bfda6c845e8bb4675
Parents: b906bdd
Author: Wangda Tan 
Authored: Tue Aug 29 15:29:22 2017 -0700
Committer: Wangda Tan 
Committed: Tue Aug 29 15:29:53 2017 -0700

--
 .../server/nodemanager/ContainerExecutor.java   |   3 +
 .../launcher/ContainerLaunch.java   |   9 ++
 .../launcher/TestContainerLaunch.java   | 105 +++
 3 files changed, 117 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bcd64d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 8357eba..5a19d2b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -330,6 +330,9 @@ public abstract class ContainerExecutor implements 
Configurable {
 whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
 whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
 
+// Add "set -o pipefail -e" to validate launch_container script.
+sb.setExitOnFailure();
+
 if (environment != null) {
   for (Map.Entry env : environment.entrySet()) {
 if (!whitelist.contains(env.getKey())) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bcd64d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index c2d8222..d29afd8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -907,6 +907,10 @@ public class ContainerLaunch implements Callable {
   sb.append(LINE_SEPARATOR);
 }
 
+public void setExitOnFailure() {
+  // Dummy implementation
+}
+
 protected abstract void link(Path src, Path dst) throws IOException;
 
 protected abstract void mkdir(Path path) throws IOException;
@@ -984,6 +988,11 @@ public class ContainerLaunch implements Callable {
   output.toString(), "\"");
   line("find -L . -maxdepth 5 -type l -ls 1>>\"", output.toString(), "\"");
 }
+
+@Override
+public void setExitOnFailure() {
+  line("set -o pipefail -e");
+}
   }
 
   private static final class WindowsShellScriptBuilder

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9bcd64d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
--
diff --git 

hadoop git commit: YARN-5219. When an export var command fails in launch_container.sh, the full container launch should fail. (Sunil G via wangda)

[wangda]

Repository: hadoop
Updated Branches:
  refs/heads/trunk ad45d1999 -> f59332b97


YARN-5219. When an export var command fails in launch_container.sh, the full 
container launch should fail. (Sunil G via wangda)

Change-Id: Iaa6b978bb89482e9d1d77ba57f4adfdc48e39a3c


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f59332b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f59332b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f59332b9

Branch: refs/heads/trunk
Commit: f59332b97b9a57e3cf1dcdeb47d7838d287100eb
Parents: ad45d19
Author: Wangda Tan 
Authored: Tue Aug 29 15:29:22 2017 -0700
Committer: Wangda Tan 
Committed: Tue Aug 29 15:29:22 2017 -0700

--
 .../server/nodemanager/ContainerExecutor.java   |   3 +
 .../launcher/ContainerLaunch.java   |   9 ++
 .../launcher/TestContainerLaunch.java   | 105 +++
 3 files changed, 117 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 9767fb9..b6fb4ec 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -337,6 +337,9 @@ public abstract class ContainerExecutor implements 
Configurable {
   whitelist.add(param);
 }
 
+// Add "set -o pipefail -e" to validate launch_container script.
+sb.setExitOnFailure();
+
 if (environment != null) {
   for (Map.Entry env : environment.entrySet()) {
 if (!whitelist.contains(env.getKey())) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 3f50c3f..50443f3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -931,6 +931,10 @@ public class ContainerLaunch implements Callable {
   sb.append(LINE_SEPARATOR);
 }
 
+public void setExitOnFailure() {
+  // Dummy implementation
+}
+
 protected abstract void link(Path src, Path dst) throws IOException;
 
 protected abstract void mkdir(Path path) throws IOException;
@@ -1008,6 +1012,11 @@ public class ContainerLaunch implements 
Callable {
   output.toString(), "\"");
   line("find -L . -maxdepth 5 -type l -ls 1>>\"", output.toString(), "\"");
 }
+
+@Override
+public void setExitOnFailure() {
+  line("set -o pipefail -e");
+}
   }
 
   private static final class WindowsShellScriptBuilder

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
 

hadoop git commit: YARN-7037. Optimize data transfer with zero-copy approach for containerlogs REST API in NMWebServices. Contributed by Tao Yang.

[junping_du]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 79294b5f3 -> 8724e3656


YARN-7037. Optimize data transfer with zero-copy approach for containerlogs 
REST API in NMWebServices. Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8724e365
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8724e365
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8724e365

Branch: refs/heads/branch-2.8
Commit: 8724e365627077aab4fa8e81f10174558fb97f9e
Parents: 79294b5
Author: Junping Du 
Authored: Tue Aug 29 15:21:56 2017 -0700
Committer: Junping Du 
Committed: Tue Aug 29 15:21:56 2017 -0700

--
 .../server/nodemanager/webapp/NMWebServices.java  | 18 --
 1 file changed, 12 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8724e365/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 02b2ab0..148f1f1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -21,6 +21,9 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
 import java.util.Map.Entry;
 
 import javax.servlet.http.HttpServletRequest;
@@ -236,17 +239,20 @@ public class NMWebServices {
 try {
   final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(
   containerIdStr, logFile, nmContext);
-  
+  final FileChannel inputChannel = fis.getChannel();
   StreamingOutput stream = new StreamingOutput() {
 @Override
 public void write(OutputStream os) throws IOException,
 WebApplicationException {
+  WritableByteChannel outputChannel = Channels.newChannel(os);
   try {
-int bufferSize = 65536;
-byte[] buf = new byte[bufferSize];
-int len;
-while ((len = fis.read(buf, 0, bufferSize)) > 0) {
-  os.write(buf, 0, len);
+long remaining = inputChannel.size();
+long position = 0;
+while (remaining > 0) {
+  long transferred =
+  inputChannel.transferTo(position, remaining, outputChannel);
+  remaining -= transferred;
+  position += transferred;
 }
 os.flush();
   } finally {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[46/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
deleted file mode 100644
index 8310530..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestLoadExampleAppJson.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.conf;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.service.api.records.Application;
-import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-
-import java.util.Arrays;
-import java.util.Collection;
-
-import static org.apache.hadoop.yarn.service.ServiceTestUtils.JSON_SER_DESER;
-import static org.easymock.EasyMock.*;
-
-/**
- * Test loading example resources.
- */
-@RunWith(value = Parameterized.class)
-public class TestLoadExampleAppJson extends Assert {
-  private String resource;
-
-  public TestLoadExampleAppJson(String resource) {
-this.resource = resource;
-  }
-
-  @Parameterized.Parameters
-  public static Collection filenames() {
-String[][] stringArray = new String[ExampleAppJson
-.ALL_EXAMPLE_RESOURCES.size()][1];
-int i = 0;
-for (String s : ExampleAppJson.ALL_EXAMPLE_RESOURCES) {
-  stringArray[i++][0] = s;
-}
-return Arrays.asList(stringArray);
-  }
-
-  @Test
-  public void testLoadResource() throws Throwable {
-try {
-  Application application = JSON_SER_DESER.fromResource(resource);
-
-  SliderFileSystem sfs = createNiceMock(SliderFileSystem.class);
-  FileSystem mockFs = createNiceMock(FileSystem.class);
-  expect(sfs.getFileSystem()).andReturn(mockFs).anyTimes();
-  expect(sfs.buildClusterDirPath(anyObject())).andReturn(
-  new Path("cluster_dir_path")).anyTimes();
-  replay(sfs, mockFs);
-
-  ServiceApiUtil.validateAndResolveApplication(application, sfs,
-  new YarnConfiguration());
-} catch (Exception e) {
-  throw new Exception("exception loading " + resource + ":" + 
e.toString());
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
deleted file mode 100644
index 98c78d3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/hadoop/yarn/service/conf/TestValidateServiceNames.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional information
- *  regarding copyright ownership.  The ASF licenses this file
- *  to you 

[13/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
deleted file mode 100644
index 3e9b764..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/publisher/PublisherResource.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.publisher;
-
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import 
org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.docstore.PublishedExportsSet;
-import org.apache.slider.core.registry.docstore.UriMap;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.AbstractSliderResource;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_REGEXP;
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.PUBLISHED_CONFIGURATION_SET_REGEXP;
-
-/**
- * This publishes configuration sets
- */
-public class PublisherResource extends AbstractSliderResource {
-  protected static final Logger log =
-  LoggerFactory.getLogger(PublisherResource.class);
-  public static final String EXPORTS_NAME = "exports";
-  public static final String EXPORTS_RESOURCES_PATH = "/" + EXPORTS_NAME;
-  public static final String EXPORT_RESOURCE_PATH = EXPORTS_RESOURCES_PATH + 
"/{exportname}" ;
-  public static final String SET_NAME =
-  "{setname: " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
-  public static final String SETNAME = "setname";
-  public static final String CLASSPATH = "/classpath";
-  public static final String CONFIG = "config";
-  
-  public static final String SETNAME_PATTERN = 
-  "{"+ SETNAME+": " + PUBLISHED_CONFIGURATION_SET_REGEXP + "}";
-  private static final String CONFIG_PATTERN =
-  SETNAME_PATTERN + "/{"+ CONFIG +": " + PUBLISHED_CONFIGURATION_REGEXP + 
"}";
-  private final StateAccessForProviders appState;
-
-  public PublisherResource(WebAppApi slider) {
-super(slider);
-appState = slider.getAppState();
-  }
-
-  private void init(HttpServletResponse res, UriInfo uriInfo) {
-res.setContentType(null);
-log.debug(uriInfo.getRequestUri().toString());
-  }
- 
-  /**
-   * Get a named config set 
-   * @param setname name of the config set
-   * @return the config set
-   * @throws NotFoundException if there was no matching set
-   */
-  private PublishedConfigSet getConfigSet(String setname) {
-PublishedConfigSet configSet =
-appState.getPublishedConfigSet(setname);
-if (configSet == null) {
-  

[69/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
new file mode 100644
index 000..fb2fd16
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -0,0 +1,654 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
+import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.service.api.ServiceApiConstants;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.component.Component;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceMetricsSink;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
+import 

[68/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
new file mode 100644
index 000..c64b1b5
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Error.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.api.records;
+
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Error {
+
+  private Integer code = null;
+  private String message = null;
+  private String fields = null;
+
+  /**
+   **/
+  public Error code(Integer code) {
+this.code = code;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("code")
+  public Integer getCode() {
+return code;
+  }
+
+  public void setCode(Integer code) {
+this.code = code;
+  }
+
+  /**
+   **/
+  public Error message(String message) {
+this.message = message;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("message")
+  public String getMessage() {
+return message;
+  }
+
+  public void setMessage(String message) {
+this.message = message;
+  }
+
+  /**
+   **/
+  public Error fields(String fields) {
+this.fields = fields;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "")
+  @JsonProperty("fields")
+  public String getFields() {
+return fields;
+  }
+
+  public void setFields(String fields) {
+this.fields = fields;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != o.getClass()) {
+  return false;
+}
+Error error = (Error) o;
+return Objects.equals(this.code, error.code)
+&& Objects.equals(this.message, error.message)
+&& Objects.equals(this.fields, error.fields);
+  }
+
+  @Override
+  public int hashCode() {
+return Objects.hash(code, message, fields);
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder();
+sb.append("class Error {\n");
+
+sb.append("code: ").append(toIndentedString(code)).append("\n");
+sb.append("message: ").append(toIndentedString(message)).append("\n");
+sb.append("fields: ").append(toIndentedString(fields)).append("\n");
+sb.append("}");
+return sb.toString();
+  }
+
+  /**
+   * Convert the given object to string with each line indented by 4 spaces
+   * (except the first line).
+   */
+  private String toIndentedString(java.lang.Object o) {
+if (o == null) {
+  return "null";
+}
+return o.toString().replace("\n", "\n");
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/PlacementPolicy.java
--
diff --git 

[18/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
deleted file mode 100644
index 2e40a9b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.rpc;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolSignature;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.proto.Messages;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-public class SliderClusterProtocolProxy implements SliderClusterProtocol {
-
-  private static final RpcController NULL_CONTROLLER = null;
-  private final SliderClusterProtocolPB endpoint;
-  private final InetSocketAddress address;
-
-  public SliderClusterProtocolProxy(SliderClusterProtocolPB endpoint,
-  InetSocketAddress address) {
-Preconditions.checkArgument(endpoint != null, "null endpoint");
-Preconditions.checkNotNull(address != null, "null address");
-this.endpoint = endpoint;
-this.address = address;
-  }
-
-  @Override
-  public String toString() {
-final StringBuilder sb =
-new StringBuilder("SliderClusterProtocolProxy{");
-sb.append("address=").append(address);
-sb.append('}');
-return sb.toString();
-  }
-
-  @Override
-  public ProtocolSignature getProtocolSignature(String protocol,
-  long clientVersion,
-  int clientMethodsHash)
-  throws IOException {
-if (!protocol.equals(RPC.getProtocolName(SliderClusterProtocolPB.class))) {
-  throw new IOException("Serverside implements " +
-RPC.getProtocolName(SliderClusterProtocolPB.class) 
+
-". The following requested protocol is unknown: " +
-protocol);
-}
-
-return ProtocolSignature.getProtocolSignature(clientMethodsHash,
-RPC.getProtocolVersion(
-SliderClusterProtocol.class),
-SliderClusterProtocol.class);
-  }
-
-  @Override
-  public long getProtocolVersion(String protocol, long clientVersion)
-  throws IOException {
-return SliderClusterProtocol.versionID;
-  }
-  
-  private IOException convert(ServiceException se) {
-IOException ioe = ProtobufHelper.getRemoteException(se);
-if (ioe instanceof RemoteException) {
-  RemoteException remoteException = (RemoteException) ioe;
-  return remoteException.unwrapRemoteException();
-}
-return ioe;
-  }
-  
-  @Override public Messages.StopClusterResponseProto stopCluster(
-  Messages.StopClusterRequestProto request)
-  throws IOException, YarnException {
-try {
-  return endpoint.stopCluster(NULL_CONTROLLER, request);
-} catch (ServiceException e) {
-  throw convert(e);
-}
-  }
-
-  @Override
-  public Messages.UpgradeContainersResponseProto upgradeContainers(
-  Messages.UpgradeContainersRequestProto request) throws IOException,
-  YarnException {
-try {
-  return endpoint.upgradeContainers(NULL_CONTROLLER, request);
-} catch (ServiceException e) {
-  throw convert(e);
-}
-  }
-
-  

[27/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
deleted file mode 100644
index fc57c82..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ /dev/null
@@ -1,2548 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.tools;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.GlobFilter;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.VersionInfo;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.NodeReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.slider.Slider;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.hadoop.yarn.service.conf.SliderKeys;
-import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
-import org.apache.hadoop.yarn.service.client.params.Arguments;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-import org.apache.slider.core.exceptions.BadClusterStateException;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.main.LauncherExitCodes;
-import org.apache.slider.server.services.utility.PatternValidator;
-import org.apache.slider.server.services.workflow.ForkedProcessService;
-import org.apache.zookeeper.server.util.KerberosUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintWriter;
-import java.io.Serializable;
-import java.io.StringWriter;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.Socket;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLDecoder;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.EnumSet;
-import 

[29/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
deleted file mode 100644
index 51bde7b..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.common.params;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import com.beust.jcommander.Parameter;
-import com.beust.jcommander.Parameters;
-import org.apache.hadoop.yarn.service.client.params.AbstractActionArgs;
-import org.apache.hadoop.yarn.service.client.params.SliderActions;
-
-@Parameters(commandNames = { SliderActions.ACTION_LIST},
-commandDescription = SliderActions.DESCRIBE_ACTION_LIST)
-
-public class ActionListArgs extends AbstractActionArgs {
-  @Override
-  public String getActionName() {
-return SliderActions.ACTION_LIST;
-  }
-
-  @Parameter(names = {ARG_LIVE},
-  description = "List only live application instances")
-  public boolean live;
-
-  @Parameter(names = {ARG_STATE},
-  description = "list only applications in the specific YARN state")
-  public String state = "";
-  
-  @Parameter(names = {ARG_VERBOSE},
-  description = "print out information in details")
-  public boolean verbose = false;
-
-  @Parameter(names = {ARG_CONTAINERS},
-  description = "List containers of an application instance")
-  public boolean containers;
-
-  @Parameter(names = {ARG_VERSION},
-  description = "Filter containers by app version (used with " +
-ARG_CONTAINERS + ")")
-  public String version;
-
-  @Parameter(names = {ARG_COMPONENTS}, variableArity = true,
-  description = "Filter containers by component names (used with " +
-ARG_CONTAINERS + ")")
-  public Set components = new HashSet<>(0);
-
-  /**
-   * Get the min #of params expected
-   * @return the min number of params in the {@link #parameters} field
-   */
-  public int getMinParams() {
-return 0;
-  }
-
-  @Override
-  public int getMaxParams() {
-return 1;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
deleted file mode 100644
index 012..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed 

[33/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
deleted file mode 100644
index e89306c..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.api.resource;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-import org.apache.commons.lang.StringUtils;
-import org.apache.slider.common.tools.SliderUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Set of configuration properties that can be injected into the application
- * components via envs, files and custom pluggable helper docker containers.
- * Files of several standard formats like xml, properties, json, yaml and
- * templates will be supported.
- **/
-
-@ApiModel(description = "Set of configuration properties that can be injected 
into the application components via envs, files and custom pluggable helper 
docker containers. Files of several standard formats like xml, properties, 
json, yaml and templates will be supported.")
-@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class Configuration implements Serializable {
-  private static final long serialVersionUID = -4330788704981074466L;
-
-  private Map properties = new HashMap();
-  private Map env = new HashMap();
-  private List files = new ArrayList();
-
-  /**
-   * A blob of key-value pairs of common application properties.
-   **/
-  public Configuration properties(Map properties) {
-this.properties = properties;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "A blob of key-value pairs of 
common application properties.")
-  @JsonProperty("properties")
-  public Map getProperties() {
-return properties;
-  }
-
-  public void setProperties(Map properties) {
-this.properties = properties;
-  }
-
-  /**
-   * A blob of key-value pairs which will be appended to the default system
-   * properties and handed off to the application at start time. All 
placeholder
-   * references to properties will be substituted before injection.
-   **/
-  public Configuration env(Map env) {
-this.env = env;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "A blob of key-value pairs which 
will be appended to the default system properties and handed off to the 
application at start time. All placeholder references to properties will be 
substituted before injection.")
-  @JsonProperty("env")
-  public Map getEnv() {
-return env;
-  }
-
-  public void setEnv(Map env) {
-this.env = env;
-  }
-
-  /**
-   * Array of list of files that needs to be created and made available as
-   * volumes in the application component containers.
-   **/
-  public Configuration files(List files) {
-this.files = files;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "Array of list of files that 
needs to be created and made available as volumes in the application component 
containers.")
-  

[39/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
new file mode 100644
index 000..e4eae20
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.containerlaunch;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryContext;
+import org.apache.hadoop.yarn.api.records.ContainerRetryPolicy;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.apache.hadoop.yarn.service.utils.CoreFileSystem;
+import org.apache.hadoop.yarn.service.utils.SliderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import static 
org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+
+/**
+ * Launcher of applications: base class
+ */
+public class AbstractLauncher {
+  private static final Logger log =
+LoggerFactory.getLogger(AbstractLauncher.class);
+  public static final String CLASSPATH = "CLASSPATH";
+  /**
+   * Filesystem to use for the launch
+   */
+  protected final CoreFileSystem coreFileSystem;
+  /**
+   * Env vars; set up at final launch stage
+   */
+  protected final Map envVars = new HashMap<>();
+  protected final ContainerLaunchContext containerLaunchContext =
+Records.newRecord(ContainerLaunchContext.class);
+  protected final List commands = new ArrayList<>(20);
+  protected final Map localResources = new HashMap<>();
+  protected final Map mountPaths = new HashMap<>();
+  private final Map serviceData = new HashMap<>();
+  // security
+  protected final Credentials credentials;
+  protected boolean yarnDockerMode = false;
+  protected String dockerImage;
+  protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
+  protected String dockerHostname;
+  protected String runPrivilegedContainer;
+
+
+  /**
+   * Create instance.
+   * @param coreFileSystem filesystem
+   * @param credentials initial set of credentials -null is permitted
+   */
+  public AbstractLauncher(
+  CoreFileSystem coreFileSystem,
+  Credentials credentials) {
+this.coreFileSystem = coreFileSystem;
+this.credentials = credentials != null ? credentials: new Credentials();
+  }
+  
+  public void setYarnDockerMode(boolean yarnDockerMode){
+this.yarnDockerMode = yarnDockerMode;
+  }
+
+  /**
+   * Get the env vars to work on
+   * @return env vars
+   */
+  public Map getEnv() {
+return envVars;
+  }
+
+  /**
+   * Get the launch commands.
+   * @return the live list of commands 
+   */
+  public List getCommands() {
+return commands;
+  }
+
+  public void addLocalResource(String subPath, LocalResource resource) {
+localResources.put(subPath, resource);
+  }
+
+  public void addLocalResource(String subPath, LocalResource resource, String 
mountPath) {
+localResources.put(subPath, 

[48/74] [abbrv] hadoop git commit: YARN-7091. Rename application to service in yarn-native-services. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
deleted file mode 100644
index 405f690..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/SerializedApplicationReport.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.service.utils;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.service.utils.ApplicationReportSerDeser;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.io.IOException;
-
-/**
- * Serialized form of an application report which can be persisted
- * and then parsed. It can not be converted back into a
- * real YARN application report
- * 
- * Useful for testing
- */
-
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-
-public class SerializedApplicationReport {
-
-  public String applicationId;
-  public String applicationAttemptId;
-  public String name;
-  public String applicationType;
-  public String user;
-  public String queue;
-  public String host;
-  public Integer rpcPort;
-  public String state;
-  public String diagnostics;
-  public String url;
-  /**
-   * This value is non-null only when a report is generated from a submission 
context.
-   * The YARN {@link ApplicationReport} structure does not propagate this value
-   * from the RM.
-   */
-  public Long submitTime;
-  public Long startTime;
-  public Long finishTime;
-  public String finalStatus;
-  public String origTrackingUrl;
-  public Float progress;
-  
-  public SerializedApplicationReport() {
-  }
-  
-  public SerializedApplicationReport(ApplicationReport report) {
-this.applicationId = report.getApplicationId().toString();
-ApplicationAttemptId attemptId = report.getCurrentApplicationAttemptId();
-this.applicationAttemptId = attemptId != null ? attemptId.toString() : 
"N/A";
-this.name = report.getName();
-this.applicationType = report.getApplicationType();
-this.user = report.getUser();
-this.queue = report.getQueue();
-this.host = report.getHost();
-this.rpcPort = report.getRpcPort();
-this.state = report.getYarnApplicationState().toString();
-this.diagnostics = report.getDiagnostics();
-this.startTime = report.getStartTime();
-this.finishTime = report.getFinishTime();
-FinalApplicationStatus appStatus = report.getFinalApplicationStatus();
-this.finalStatus = appStatus == null ? "" : appStatus.toString();
-this.progress = report.getProgress();
-this.url = report.getTrackingUrl();
-this.origTrackingUrl= report.getOriginalTrackingUrl();
-  }
-
-  @Override
-  public String toString() {
-try {
-  return ApplicationReportSerDeser.toString(this);
-} catch (IOException e) {
-  return super.toString();
-}
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c07469f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
--
diff --git 

[74/74] [abbrv] hadoop git commit: YARN-7113. Clean up packaging and dependencies for yarn-native-services. Contributed by Billie Rinaldi

[jianhe]

YARN-7113. Clean up packaging and dependencies for yarn-native-services. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54ad59a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54ad59a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54ad59a4

Branch: refs/heads/yarn-native-services
Commit: 54ad59a4745dc5c6d150d58127a2fe174d1e38b8
Parents: b5ec754
Author: Jian He 
Authored: Tue Aug 29 11:09:00 2017 -0700
Committer: Jian He 
Committed: Tue Aug 29 11:31:15 2017 -0700

--
 NOTICE.txt  |   15 +
 .../resources/assemblies/hadoop-yarn-dist.xml   |8 -
 .../assemblies/hadoop-yarn-services-api.xml |   36 -
 .../assemblies/hadoop-yarn-services-dist.xml|   30 -
 hadoop-project/pom.xml  |   19 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   37 +-
 .../hadoop-yarn-services-api/pom.xml|  104 +-
 .../yarn/service/webapp/ApiServerWebApp.java|4 +-
 .../src/main/resources/webapps/api-server/app   |   16 +
 .../resources/webapps/services-rest-api/app |   16 -
 .../hadoop-yarn-services-core/pom.xml   |  213 +---
 .../service/client/params/ActionKDiagArgs.java  |   76 --
 .../yarn/service/client/params/ClientArgs.java  |5 -
 .../registry/YarnRegistryViewForProviders.java  |8 +-
 .../yarn/service/utils/KerberosDiags.java   |  680 ---
 .../hadoop/yarn/service/utils/SliderUtils.java  | 1088 --
 .../hadoop/yarn/service/ServiceTestUtils.java   |   28 +
 .../hadoop/yarn/service/TestServiceApiUtil.java |   38 +-
 .../yarn/service/TestYarnNativeServices.java|   10 +-
 .../yarn/service/client/TestServiceCLI.java |1 -
 .../yarn/service/conf/TestAppJsonResolve.java   |   30 +-
 .../service/conf/TestLoadExampleAppJson.java|   11 +-
 .../providers/TestAbstractClientProvider.java   |   10 +-
 hadoop-yarn-project/pom.xml |4 +
 24 files changed, 176 insertions(+), 2311 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/54ad59a4/NOTICE.txt
--
diff --git a/NOTICE.txt b/NOTICE.txt
index c41972b..e40f3bf 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -576,3 +576,18 @@ The binary distribution of this product bundles binaries of
 software.amazon.ion:ion-java 1.0.1,
 which has the following notices:
  * Amazon Ion Java Copyright 2007-2016 Amazon.com, Inc. or its affiliates. All 
Rights Reserved.
+
+The binary distribution of this product bundles binaries of
+JCommander (https://github.com/cbeust/jcommander),
+which has the following notices:
+ * Copyright 2010 Cedric Beust ced...@beust.com
+
+The binary distribution of this product bundles binaries of
+snakeyaml (https://bitbucket.org/asomov/snakeyaml),
+which has the following notices:
+ * Copyright (c) 2008, http://www.snakeyaml.org
+
+The binary distribution of this product bundles binaries of
+swagger-annotations (https://github.com/swagger-api/swagger-core),
+which has the following notices:
+ * Copyright 2016 SmartBear Software

http://git-wip-us.apache.org/repos/asf/hadoop/blob/54ad59a4/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 8aeeabd..8b3d292 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -98,10 +98,6 @@
   etc/hadoop
 
 
-  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/target/hadoop-yarn-services-core-${project.version}
-  
/share/hadoop/${hadoop.component}/lib/services
-
-
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
   
/share/hadoop/${hadoop.component}/sources
   
@@ -109,10 +105,6 @@
   
 
 
-  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}
-  
/share/hadoop/${hadoop.component}/lib/services-api
-
-
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/54ad59a4/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml 

[20/74] [abbrv] hadoop git commit: YARN-7050. Post cleanup after YARN-6903, removal of org.apache.slider package. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
deleted file mode 100644
index 7446e82..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.actions;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.slider.server.appmaster.SliderAppMaster;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
-import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
-import org.apache.slider.server.appmaster.operations.RMOperationHandler;
-import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-import org.apache.slider.server.appmaster.state.AppState;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Kill a specific container
- */
-public class ActionKillContainer extends AsyncAction {
-
-  /**
-   *  container to kill
-   */
-  private final ContainerId containerId;
-
-  /**
-   *  handler for the operation
-   */
-  private final RMOperationHandlerActions operationHandler;
-
-  /**
-   * Kill a container
-   * @param containerId container to kill
-   * @param delay
-   * @param timeUnit
-   * @param operationHandler
-   */
-  public ActionKillContainer(
-  ContainerId containerId,
-  long delay,
-  TimeUnit timeUnit,
-  RMOperationHandlerActions operationHandler) {
-super("kill container", delay, timeUnit, ATTR_CHANGES_APP_SIZE);
-this.operationHandler = operationHandler;
-Preconditions.checkArgument(containerId != null);
-
-this.containerId = containerId;
-  }
-
-  /**
-   * Get the container ID to kill
-   * @return
-   */
-  public ContainerId getContainerId() {
-return containerId;
-  }
-
-  @Override
-  public void execute(SliderAppMaster appMaster,
-  QueueAccess queueService,
-  AppState appState) throws Exception {
-  List opsList = new LinkedList<>();
-ContainerReleaseOperation release = new 
ContainerReleaseOperation(containerId);
-opsList.add(release);
-//now apply the operations
-operationHandler.execute(opsList);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/43277ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
deleted file mode 100644
index 0d7f7d4..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionRegisterServiceInstance.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional