hadoop git commit: HDFS-11531. Expose hedged read metrics via libHDFS API. Contributed by Sailesh Mukil.

[jzhuge]

Repository: hadoop
Updated Branches:
  refs/heads/trunk af8e9842d -> 8c81a16a1


HDFS-11531. Expose hedged read metrics via libHDFS API. Contributed by Sailesh 
Mukil.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c81a16a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c81a16a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c81a16a

Branch: refs/heads/trunk
Commit: 8c81a16a1fa40d3a3796b1c3633328f1340883ca
Parents: af8e984
Author: John Zhuge 
Authored: Tue Apr 18 21:11:07 2017 -0700
Committer: John Zhuge 
Committed: Tue Apr 18 21:11:07 2017 -0700

--
 .../hadoop/hdfs/DistributedFileSystem.java  |  9 +++
 .../libhdfs-tests/test_libhdfs_threaded.c   | 10 +++
 .../src/main/native/libhdfs/hdfs.c  | 81 
 .../src/main/native/libhdfs/include/hdfs/hdfs.h | 29 +++
 4 files changed, 129 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 11d7eb8..e1c2105 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -201,6 +201,15 @@ public class DistributedFileSystem extends FileSystem {
   }
 
   /**
+   * Returns the hedged read metrics object for this client.
+   *
+   * @return object of DFSHedgedReadMetrics
+   */
+  public DFSHedgedReadMetrics getHedgedReadMetrics() {
+return dfs.getHedgedReadMetrics();
+  }
+
+  /**
* Checks that the passed URI belongs to this filesystem and returns
* just the path component. Expects a URI with an absolute path.
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
 
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
index 032acbf..ee1b080 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs-tests/test_libhdfs_threaded.c
@@ -145,6 +145,7 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, 
hdfsFS fs,
 int ret, expected, numEntries;
 hdfsFileInfo *fileInfo;
 struct hdfsReadStatistics *readStats = NULL;
+struct hdfsHedgedReadMetrics *hedgedMetrics = NULL;
 
 if (hdfsExists(fs, paths->prefix) == 0) {
 EXPECT_ZERO(hdfsDelete(fs, paths->prefix, 1));
@@ -204,6 +205,15 @@ static int doTestHdfsOperations(struct tlhThreadInfo *ti, 
hdfsFS fs,
 EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalLocalBytesRead);
 EXPECT_UINT64_EQ(UINT64_C(0), readStats->totalShortCircuitBytesRead);
 hdfsFileFreeReadStatistics(readStats);
+
+/* Verify that we can retrieve the hedged read metrics */
+EXPECT_ZERO(hdfsGetHedgedReadMetrics(fs, ));
+errno = 0;
+EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOps);
+EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOpsWin);
+EXPECT_UINT64_EQ(UINT64_C(0), hedgedMetrics->hedgedReadOpsInCurThread);
+hdfsFreeHedgedReadMetrics(hedgedMetrics);
+
 /* TODO: implement readFully and use it here */
 ret = hdfsRead(fs, file, tmp, sizeof(tmp));
 if (ret < 0) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c81a16a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c 
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
index 1dcc768..36c9583 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c
@@ -93,6 +93,87 @@ int hdfsFileIsOpenForRead(hdfsFile file)
 return (file->type == HDFS_STREAM_INPUT);
 }
 
+int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics)
+{
+jthrowable jthr;
+jobject hedgedReadMetrics = NULL;
+jvalue 

hadoop git commit: HDFS-11649. Ozone: SCM: CLI: Add shell code placeholder classes. Contributed by Chen Liang.

[wwei]

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 d889437c2 -> 0bf416ac4


HDFS-11649. Ozone: SCM: CLI: Add shell code placeholder classes. Contributed by 
Chen Liang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0bf416ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0bf416ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0bf416ac

Branch: refs/heads/HDFS-7240
Commit: 0bf416ac4004d0fe8418f1a79e64c8ae1f796384
Parents: d889437
Author: Weiwei Yang 
Authored: Wed Apr 19 11:43:14 2017 +0800
Committer: Weiwei Yang 
Committed: Wed Apr 19 11:43:14 2017 +0800

--
 .../ozone/scm/StorageContainerManager.java  |   5 +
 .../hadoop/ozone/scm/cli/OzoneBaseCLI.java  |  44 
 .../ozone/scm/cli/OzoneCommandHandler.java  |  60 +
 .../apache/hadoop/ozone/scm/cli/ResultCode.java |  31 +++
 .../org/apache/hadoop/ozone/scm/cli/SCMCLI.java | 237 +++
 .../cli/container/ContainerCommandHandler.java  |  92 +++
 .../cli/container/CreateContainerHandler.java   |  78 ++
 .../ozone/scm/cli/container/package-info.java   |  19 ++
 .../hadoop/ozone/scm/cli/package-info.java  |  19 ++
 .../org/apache/hadoop/ozone/scm/TestSCMCli.java | 155 
 10 files changed, 740 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bf416ac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
index bdc0418..f3112c9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
@@ -357,6 +357,11 @@ public class StorageContainerManager
 ScmClient.ReplicationFactor.ONE);
   }
 
+  @VisibleForTesting
+  Pipeline getContainer(String containerName) throws IOException {
+return scmContainerManager.getContainer(containerName);
+  }
+
   /**
* Asks SCM where a container should be allocated. SCM responds with the set
* of datanodes that should be used creating this container.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0bf416ac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
new file mode 100644
index 000..fdd0a49
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.scm.cli;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.ozone.web.exceptions.OzoneException;
+import org.apache.hadoop.util.Tool;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+
+/**
+ * This class is the base CLI for scm, ksm and scmadm.
+ */
+public abstract class OzoneBaseCLI extends Configured implements Tool {
+
+  protected abstract int dispatch(CommandLine cmd, Options opts)
+  throws IOException, OzoneException, URISyntaxException;
+
+  protected abstract CommandLine parseArgs(String[] argv, Options opts)
+  throws ParseException;
+
+  protected abstract Options getOptions();
+
+  protected abstract void displayHelp();
+}

hadoop git commit: HDFS-11603. Improve slow mirror/disk warnings in BlockReceiver.

[arp]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6534bc0a0 -> 5c092097b


HDFS-11603. Improve slow mirror/disk warnings in BlockReceiver.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c092097
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c092097
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c092097

Branch: refs/heads/branch-2
Commit: 5c092097b6481df6494fc0bb52374971a28636d8
Parents: 6534bc0
Author: Arpit Agarwal 
Authored: Tue Apr 18 16:20:43 2017 -0700
Committer: Arpit Agarwal 
Committed: Tue Apr 18 16:20:43 2017 -0700

--
 .../hadoop/hdfs/protocol/DatanodeInfo.java  |  1 +
 .../hdfs/server/datanode/BlockReceiver.java | 62 +++-
 .../hdfs/server/datanode/ReplicaHandler.java|  5 ++
 3 files changed, 53 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c092097/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index e1698c9..0a8c915 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -55,6 +55,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
   private String softwareVersion;
   private List dependentHostNames = new LinkedList<>();
   private String upgradeDomain;
+  public static final DatanodeInfo[] EMPTY_ARRAY = {};
 
   // Datanode administrative states
   public enum AdminStates {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c092097/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 11ae7b0..81945f9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -45,6 +45,7 @@ import 
org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver;
 import org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck;
 import 
org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.BlockOpResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.Status;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaInputStreams;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.ReplicaOutputStreams;
 import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodePeerMetrics;
@@ -100,6 +101,7 @@ class BlockReceiver implements Closeable {
   private DataTransferThrottler throttler;
   private ReplicaOutputStreams streams;
   private DatanodeInfo srcDataNode = null;
+  private DatanodeInfo[] downstreamDNs = DatanodeInfo.EMPTY_ARRAY;
   private final DataNode datanode;
   volatile private boolean mirrorError;
 
@@ -424,10 +426,10 @@ class BlockReceiver implements Closeable {
   }
 }
 long duration = Time.monotonicNow() - begin;
-if (duration > datanodeSlowLogThresholdMs) {
+if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
   LOG.warn("Slow flushOrSync took " + duration + "ms (threshold="
   + datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", 
flushTotalNanos="
-  + flushTotalNanos + "ns");
+  + flushTotalNanos + "ns, volume=" + getVolumeBasePath());
 }
   }
 
@@ -578,9 +580,10 @@ class BlockReceiver implements Closeable {
 mirrorAddr,
 duration);
 trackSendPacketToLastNodeInPipeline(duration);
-if (duration > datanodeSlowLogThresholdMs) {
+if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
   LOG.warn("Slow BlockReceiver write packet to mirror took " + duration
-  + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms)");
+  + "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), "
+  + "downstream DNs=" + Arrays.toString(downstreamDNs));
 }
   } catch (IOException e) {
 handleMirrorOutError(e);
@@ -711,9 +714,10 @@ 

[2/2] hadoop git commit: HDFS-11551. Handle SlowDiskReport from DataNode at the NameNode. Contributed by Hanisha Koneru.

[arp]

HDFS-11551. Handle SlowDiskReport from DataNode at the NameNode. Contributed by 
Hanisha Koneru.

This is an empty commit with the correct commit message for HDFS-11511.
The actual fix was made via commit ID f16bb799573bea6da2535ee863ed15ca271ef504.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6534bc0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6534bc0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6534bc0a

Branch: refs/heads/branch-2
Commit: 6534bc0a01220ae90049b02ff7b94c1e8a47bff5
Parents: fc358b8
Author: Arpit Agarwal 
Authored: Tue Apr 18 15:35:17 2017 -0700
Committer: Arpit Agarwal 
Committed: Tue Apr 18 15:35:17 2017 -0700

--

--



-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[1/2] hadoop git commit: HDFS-11511. Support Timeout when checking single disk. Contributed by Hanisha Koneru.

[arp]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 6cfceee86 -> 6534bc0a0


HDFS-11511. Support Timeout when checking single disk. Contributed by Hanisha 
Koneru.

This is an empty commit with the correct commit message for HDFS-11511.
The fix was previously made via commit ID 
fbe86471cf6d6397a89e473eb328d86a66c513e2.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc358b89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc358b89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc358b89

Branch: refs/heads/branch-2
Commit: fc358b89b8b541f8e2d22e1948d457debe07ec53
Parents: 6cfceee
Author: Arpit Agarwal 
Authored: Tue Apr 18 15:34:46 2017 -0700
Committer: Arpit Agarwal 
Committed: Tue Apr 18 15:34:46 2017 -0700

--

--



-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HDFS-11334: [SPS]: NN switch and rescheduling movements can lead to have more than one coordinator for same file blocks. Contributed by Rakesh R.

[umamahesh]

Repository: hadoop
Updated Branches:
  refs/heads/HDFS-10285 6efbd207e -> b1fb932b5


HDFS-11334: [SPS]: NN switch and rescheduling movements can lead to have more 
than one coordinator for same file blocks. Contributed by Rakesh R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1fb932b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1fb932b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1fb932b

Branch: refs/heads/HDFS-10285
Commit: b1fb932b5159b878f236db56066aca3d7e4b64dc
Parents: 6efbd20
Author: Uma Maheswara Rao G 
Authored: Tue Apr 18 15:23:58 2017 -0700
Committer: Uma Maheswara Rao G 
Committed: Tue Apr 18 15:23:58 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   2 +-
 .../apache/hadoop/hdfs/protocolPB/PBHelper.java |   6 +
 .../server/blockmanagement/DatanodeManager.java |  12 ++
 .../hdfs/server/datanode/BPServiceActor.java|   4 +-
 .../datanode/BlockStorageMovementTracker.java   |  37 +++-
 .../hadoop/hdfs/server/datanode/DataNode.java   |  12 +-
 .../datanode/StoragePolicySatisfyWorker.java|  95 +--
 .../BlockStorageMovementAttemptedItems.java |  80 ++---
 .../server/namenode/StoragePolicySatisfier.java |  15 +-
 .../protocol/BlocksStorageMovementResult.java   |   6 +-
 .../src/main/proto/DatanodeProtocol.proto   |   1 +
 .../TestStoragePolicySatisfyWorker.java |  68 
 .../TestStoragePolicySatisfierWithHA.java   | 170 +--
 13 files changed, 413 insertions(+), 95 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fb932b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 7154bb4..3ed332b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -554,7 +554,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final String 
DFS_STORAGE_POLICY_SATISFIER_SELF_RETRY_TIMEOUT_MILLIS_KEY =
   "dfs.storage.policy.satisfier.self.retry.timeout.millis";
   public static final int 
DFS_STORAGE_POLICY_SATISFIER_SELF_RETRY_TIMEOUT_MILLIS_DEFAULT =
-  30 * 60 * 1000;
+  20 * 60 * 1000;
 
   public static final String  DFS_DATANODE_ADDRESS_KEY = 
"dfs.datanode.address";
   public static final int DFS_DATANODE_DEFAULT_PORT = 9866;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fb932b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 173f954..2e87e04 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -913,6 +913,9 @@ public class PBHelper {
   case FAILURE:
 status = Status.FAILURE;
 break;
+  case IN_PROGRESS:
+status = Status.IN_PROGRESS;
+break;
   default:
 throw new AssertionError("Unknown status: " + resultProto.getStatus());
   }
@@ -939,6 +942,9 @@ public class PBHelper {
   case FAILURE:
 status = BlocksStorageMovementResultProto.Status.FAILURE;
 break;
+  case IN_PROGRESS:
+status = BlocksStorageMovementResultProto.Status.IN_PROGRESS;
+break;
   default:
 throw new AssertionError("Unknown status: " + report.getStatus());
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1fb932b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index b15b6ac..a240af3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ 

hadoop git commit: Update CHANGES.txt for HDFS-11634.

[shv]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 5a714fee4 -> 54bbdb4c0


Update CHANGES.txt for HDFS-11634.

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54bbdb4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54bbdb4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54bbdb4c

Branch: refs/heads/branch-2.7
Commit: 54bbdb4c026092be47110f2f90d40c14c0e05c68
Parents: 5a714fe
Author: Konstantin V Shvachko 
Authored: Tue Apr 18 13:20:21 2017 -0700
Committer: Konstantin V Shvachko 
Committed: Tue Apr 18 13:20:21 2017 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/54bbdb4c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 286b3a0..33544e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -80,6 +80,9 @@ Release 2.7.4 - UNRELEASED
 HDFS-10896. Move lock logging logic from FSNamesystem into 
FSNamesystemLock.
 (Erik Krogen via Zhe Zhang)
 
+HDFS-11634. Optimize BlockIterator when iterating starts in the middle.
+(shv)
+
   BUG FIXES

 HDFS-8307. Spurious DNS Queries from hdfs shell. (Andres Perez via 
aengineer)


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6451. Add RM monitor validating metrics invariants. Contributed by Carlo Curino

[cdouglas]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a9f07e0d3 -> af8e9842d


YARN-6451. Add RM monitor validating metrics invariants. Contributed by Carlo 
Curino


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af8e9842
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af8e9842
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af8e9842

Branch: refs/heads/trunk
Commit: af8e9842d2ca566528e09d905b609f1cf160d367
Parents: a9f07e0
Author: Chris Douglas 
Authored: Tue Apr 18 10:28:50 2017 -0700
Committer: Chris Douglas 
Committed: Tue Apr 18 10:29:44 2017 -0700

--
 .../hadoop-yarn-server-resourcemanager/pom.xml  |   1 +
 .../invariants/InvariantViolationException.java |  35 
 .../monitor/invariants/InvariantsChecker.java   |  96 +
 .../invariants/MetricsInvariantChecker.java | 195 +++
 .../monitor/invariants/package-info.java|  22 +++
 .../invariants/TestMetricsInvariantChecker.java |  99 ++
 .../src/test/resources/invariants.txt   |  54 +
 7 files changed, 502 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/af8e9842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 533be9e..8505df8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -347,6 +347,7 @@
 src/test/resources/submit-reservation.json
 src/test/resources/delete-reservation.json
 src/test/resources/update-reservation.json
+src/test/resources/invariants.txt
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af8e9842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/InvariantViolationException.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/InvariantViolationException.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/InvariantViolationException.java
new file mode 100644
index 000..0491756
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/InvariantViolationException.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.invariants;
+
+
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+/**
+ * This exception represents the violation of an internal invariant.
+ */
+public class InvariantViolationException extends YarnRuntimeException {
+
+  public InvariantViolationException(String s) {
+super(s);
+  }
+
+  public InvariantViolationException(String s, Exception e) {
+super(s, e);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/af8e9842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/invariants/InvariantsChecker.java
--
diff --git 

hadoop git commit: HDFS-11634. Optimize BlockIterator when interating starts in the middle. Contributed by Konstantin V Shvachko.

[shv]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 ccd24f091 -> 5a714fee4


HDFS-11634. Optimize BlockIterator when interating starts in the middle. 
Contributed by Konstantin V Shvachko.

(cherry picked from commit 8dfcd95d580bb090af7f40af0a57061518c18c8c)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetBlocks.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a714fee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a714fee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a714fee

Branch: refs/heads/branch-2.7
Commit: 5a714fee4376821c9ce1a5e00c34b3acc25240e7
Parents: ccd24f0
Author: Konstantin V Shvachko 
Authored: Mon Apr 17 15:04:06 2017 -0700
Committer: Konstantin V Shvachko 
Committed: Mon Apr 17 17:39:29 2017 -0700

--
 .../server/blockmanagement/BlockManager.java|  7 +-
 .../blockmanagement/DatanodeDescriptor.java | 33 +++--
 .../org/apache/hadoop/hdfs/TestGetBlocks.java   | 78 ++--
 .../blockmanagement/BlockManagerTestUtil.java   | 16 
 4 files changed, 115 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a714fee/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index cc6c881..85f9201 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1060,12 +1060,9 @@ public class BlockManager {
 if(numBlocks == 0) {
   return new BlocksWithLocations(new BlockWithLocations[0]);
 }
-Iterator iter = node.getBlockIterator();
+// starting from a random block
 int startBlock = DFSUtil.getRandom().nextInt(numBlocks); // starting from 
a random block
-// skip blocks
-for(int i=0; i iterators;
 
-private BlockIterator(final DatanodeStorageInfo... storages) {
+private BlockIterator(final int startBlock,
+  final DatanodeStorageInfo... storages) {
+  if(startBlock < 0) {
+throw new IllegalArgumentException(
+"Illegal value startBlock = " + startBlock);
+  }
   List iterators = new 
ArrayList();
+  int s = startBlock;
+  int sumBlocks = 0;
   for (DatanodeStorageInfo e : storages) {
-iterators.add(e.getBlockIterator());
+int numBlocks = e.numBlocks();
+sumBlocks += numBlocks;
+if(sumBlocks <= startBlock) {
+  s -= numBlocks;
+} else {
+  iterators.add(e.getBlockIterator());
+}
   }
   this.iterators = Collections.unmodifiableList(iterators);
+  // skip to the storage containing startBlock
+  for(; s > 0 && hasNext(); s--) {
+next();
+  }
 }
 
 @Override
 public boolean hasNext() {
   update();
-  return !iterators.isEmpty() && iterators.get(index).hasNext();
+  return index < iterators.size() && 

[1/2] hadoop git commit: HDFS-11665. HttpFSServerWebServer$deprecateEnv may leak secret. Contributed by John Zhuge.

[jzhuge]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 654372db8 -> a9f07e0d3


HDFS-11665. HttpFSServerWebServer$deprecateEnv may leak secret. Contributed by 
John Zhuge.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca2488c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca2488c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca2488c0

Branch: refs/heads/trunk
Commit: ca2488c009b42fd45531af60b70872e4323de20b
Parents: 654372d
Author: John Zhuge 
Authored: Mon Apr 17 22:36:20 2017 -0700
Committer: John Zhuge 
Committed: Tue Apr 18 09:28:52 2017 -0700

--
 .../apache/hadoop/fs/http/server/HttpFSServerWebServer.java   | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca2488c0/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java
 
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java
index 7bda965..6bd0d12 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServerWebServer.java
@@ -123,10 +123,9 @@ public class HttpFSServerWebServer {
 if (value == null) {
   return;
 }
-String propValue = conf.get(propName);
-LOG.warn("Environment variable {} = '{}' is deprecated and overriding"
-+ " property {} = '{}', please set the property in {} instead.",
-varName, value, propName, propValue, confFile);
+LOG.warn("Environment variable {} is deprecated and overriding"
++ " property {}', please set the property in {} instead.",
+varName, propName, confFile);
 conf.set(propName, value, "environment variable " + varName);
   }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[2/2] hadoop git commit: HADOOP-14317. KMSWebServer$deprecateEnv may leak secret. Contributed by John Zhuge.

[jzhuge]

HADOOP-14317. KMSWebServer$deprecateEnv may leak secret. Contributed by John 
Zhuge.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9f07e0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9f07e0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9f07e0d

Branch: refs/heads/trunk
Commit: a9f07e0d3ebb41d24d11e2bdb0ee872fa72072ca
Parents: ca2488c
Author: John Zhuge 
Authored: Mon Apr 17 22:35:50 2017 -0700
Committer: John Zhuge 
Committed: Tue Apr 18 09:29:16 2017 -0700

--
 .../org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9f07e0d/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
--
diff --git 
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
 
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
index 1141824..ced1f69 100644
--- 
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
+++ 
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebServer.java
@@ -105,10 +105,9 @@ public class KMSWebServer {
 if (value == null) {
   return;
 }
-String propValue = conf.get(propName);
-LOG.warn("Environment variable {} = '{}' is deprecated and overriding"
-+ " property {} = '{}', please set the property in {} instead.",
-varName, value, propName, propValue, confFile);
+LOG.warn("Environment variable {} is deprecated and overriding"
++ " property {}, please set the property in {} instead.",
+varName, propName, confFile);
 conf.set(propName, value, "environment variable " + varName);
   }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[Hadoop Wiki] Update of "S3ABadRequest" by SteveLoughran

[Apache Wiki]

Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change 
notification.

The "S3ABadRequest" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/S3ABadRequest

Comment:
Initial set of Bad Request causes

New page:
a5a4867f3b HADOOP-14120

= Troubleshooting S3A Bad Request Errors =

The S3A client can see the error message "Bad Request" for many reasons —it is 
the standard response from
Amazon S3 if it could not satisfy the request *for any reason*.

The main issues are covered in the 
[[http://hadoop.apache.org/docs/current//hadoop-aws/tools/hadoop-aws/index.html#Troubleshooting_S3A|Troubleshooting
 S3A]] section of the hadoop-aws module's documentation.


== Common Causes of Bad Request Error Messages ==

=== Credentials ===

 * Your credentials are wrong.
 * Somehow the credentials have not been set properly before the S3A Filesystem 
instance was created. As a single instance per bucket is created per-JVM, the 
first configuration used to connect to a bucket is the one used thereafter.
 * You've been trying to set the credentials in the URI, but got the 
URL-escaping wrong. Stop trying to do that, it's a security disaster. Embrace 
per-bucket configuration.
 * You are trying to use per-bucket configuration for the credentials, but got 
the bucket name wrong there.
 * You are using session credentials, and the session has expired.


=== Endpoints ===

 * You are trying to use a V4 auth endpoint without declaring the endpoint of 
that region in the {{{fs.s3a.endpoint}}}.
 * You are trying to use a V3 auth endpoint but have set up S3 to use an 
explicit V4 auth endpoint. As they do not redirect to the central endpoint, you 
must declare the relevant endpoint explicitly.
 * You are trying to use a private S3 service but have forgotten to set the 
{{{fs.s3a.endpoint}}}; AWS is rejecting your private login.
 * You are trying to talk to a private S3 service but somehow it is talking to 
an HTTP page rather than an implementation of the S3 REST API.

=== Encryption ===

 * You are trying to use SSE-C with a key that cannot decrypt the remote data.
 * You are trying to work with a bucket which is configured to require 
encryption, but the client doesn't use it.

=== Classpath ===

 * A version of Joda-time incompatible with the JVM is on the classpath. It 
must be version 2.9.1 or later.

=== System ===

 * The client machine doesn't know when it is. Check the clock and the timezone 
settings.
 * Your DNS setup is returning the wrong IP address for the endpoint.
 * Your network is a mess.

As you can see, there is a wide variety of possible causes, spread across: 
credential setup, endpoint configuration, system configuration and other 
aspects of the S3A client. We are hampered in helping diagnose this by the need 
to keep those credentials secret.

== Logging at lower levels ==

The AWS SDK and the Apache HTTP components can be configured to log at
more detail, as can S3A itself.

{{{
log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
log4j.logger.com.amazonaws.request=DEBUG
log4j.logger.org.apache.http=DEBUG
log4j.logger.org.apache.http.wire=ERROR
}}}

Be aware that logging HTTP headers may leak sensitive AWS account information,
so the output should not be shared.

-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14315. Python example in the rack awareness document doesn't work due to bad indentation

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 c8bcc9650 -> 525b25894


HADOOP-14315. Python example in the rack awareness document doesn't work due to 
bad indentation

This closes #214

Signed-off-by: Akira Ajisaka 
(cherry picked from commit 654372db859656a2201ae9f9f7c374c6564ea34d)
(cherry picked from commit 6cfceee86547211c71f8cb6f9c63798a229de03e)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/525b2589
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/525b2589
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/525b2589

Branch: refs/heads/branch-2.8
Commit: 525b2589444b33ba65c618b6404e79e8e3e38bbb
Parents: c8bcc96
Author: Kengo Seki 
Authored: Mon Apr 17 23:29:58 2017 -0400
Committer: Akira Ajisaka 
Committed: Tue Apr 18 17:16:03 2017 +0900

--
 .../hadoop-common/src/site/markdown/RackAwareness.md| 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/525b2589/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
index ced6c84..0ae1de3 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -91,12 +91,12 @@ sys.argv.pop(0) 
 # discard name
 netmask = '255.255.255.0'# set netmask 
to what's being used in your environment.  The example uses a /24
 
 for ip in sys.argv:  # loop over 
list of datanode IP's
-address = '{0}/{1}'.format(ip, netmask)  # format address 
string so it looks like 'ip/netmask' to make netaddr work
-try:
-   network_address = netaddr.IPNetwork(address).network # calculate and 
print network address
-   print "/{0}".format(network_address)
-except:
-   print "/rack-unknown"# print catch-all 
value if unable to calculate network address
+address = '{0}/{1}'.format(ip, netmask)  # format 
address string so it looks like 'ip/netmask' to make netaddr work
+try:
+network_address = netaddr.IPNetwork(address).network # calculate 
and print network address
+print "/{0}".format(network_address)
+except:
+print "/rack-unknown"# print 
catch-all value if unable to calculate network address
 ```
 
 bash Example


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14315. Python example in the rack awareness document doesn't work due to bad indentation

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8.1 15c6d6847 -> 2a18511b7


HADOOP-14315. Python example in the rack awareness document doesn't work due to 
bad indentation

This closes #214

Signed-off-by: Akira Ajisaka 
(cherry picked from commit 654372db859656a2201ae9f9f7c374c6564ea34d)
(cherry picked from commit 6cfceee86547211c71f8cb6f9c63798a229de03e)
(cherry picked from commit 525b2589444b33ba65c618b6404e79e8e3e38bbb)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a18511b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a18511b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a18511b

Branch: refs/heads/branch-2.8.1
Commit: 2a18511b7092bd51914842e7e2dc9a10119a2d65
Parents: 15c6d68
Author: Kengo Seki 
Authored: Mon Apr 17 23:29:58 2017 -0400
Committer: Akira Ajisaka 
Committed: Tue Apr 18 17:16:21 2017 +0900

--
 .../hadoop-common/src/site/markdown/RackAwareness.md| 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a18511b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
index ced6c84..0ae1de3 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -91,12 +91,12 @@ sys.argv.pop(0) 
 # discard name
 netmask = '255.255.255.0'# set netmask 
to what's being used in your environment.  The example uses a /24
 
 for ip in sys.argv:  # loop over 
list of datanode IP's
-address = '{0}/{1}'.format(ip, netmask)  # format address 
string so it looks like 'ip/netmask' to make netaddr work
-try:
-   network_address = netaddr.IPNetwork(address).network # calculate and 
print network address
-   print "/{0}".format(network_address)
-except:
-   print "/rack-unknown"# print catch-all 
value if unable to calculate network address
+address = '{0}/{1}'.format(ip, netmask)  # format 
address string so it looks like 'ip/netmask' to make netaddr work
+try:
+network_address = netaddr.IPNetwork(address).network # calculate 
and print network address
+print "/{0}".format(network_address)
+except:
+print "/rack-unknown"# print 
catch-all value if unable to calculate network address
 ```
 
 bash Example


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14315. Python example in the rack awareness document doesn't work due to bad indentation

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 a17bd7089 -> 6cfceee86


HADOOP-14315. Python example in the rack awareness document doesn't work due to 
bad indentation

This closes #214

Signed-off-by: Akira Ajisaka 
(cherry picked from commit 654372db859656a2201ae9f9f7c374c6564ea34d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6cfceee8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6cfceee8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6cfceee8

Branch: refs/heads/branch-2
Commit: 6cfceee86547211c71f8cb6f9c63798a229de03e
Parents: a17bd70
Author: Kengo Seki 
Authored: Mon Apr 17 23:29:58 2017 -0400
Committer: Akira Ajisaka 
Committed: Tue Apr 18 17:15:35 2017 +0900

--
 .../hadoop-common/src/site/markdown/RackAwareness.md| 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6cfceee8/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
index ced6c84..0ae1de3 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -91,12 +91,12 @@ sys.argv.pop(0) 
 # discard name
 netmask = '255.255.255.0'# set netmask 
to what's being used in your environment.  The example uses a /24
 
 for ip in sys.argv:  # loop over 
list of datanode IP's
-address = '{0}/{1}'.format(ip, netmask)  # format address 
string so it looks like 'ip/netmask' to make netaddr work
-try:
-   network_address = netaddr.IPNetwork(address).network # calculate and 
print network address
-   print "/{0}".format(network_address)
-except:
-   print "/rack-unknown"# print catch-all 
value if unable to calculate network address
+address = '{0}/{1}'.format(ip, netmask)  # format 
address string so it looks like 'ip/netmask' to make netaddr work
+try:
+network_address = netaddr.IPNetwork(address).network # calculate 
and print network address
+print "/{0}".format(network_address)
+except:
+print "/rack-unknown"# print 
catch-all value if unable to calculate network address
 ```
 
 bash Example


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14315. Python example in the rack awareness document doesn't work due to bad indentation

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 84a8848aa -> 654372db8


HADOOP-14315. Python example in the rack awareness document doesn't work due to 
bad indentation

This closes #214

Signed-off-by: Akira Ajisaka 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/654372db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/654372db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/654372db

Branch: refs/heads/trunk
Commit: 654372db859656a2201ae9f9f7c374c6564ea34d
Parents: 84a8848
Author: Kengo Seki 
Authored: Mon Apr 17 23:29:58 2017 -0400
Committer: Akira Ajisaka 
Committed: Tue Apr 18 17:14:22 2017 +0900

--
 .../hadoop-common/src/site/markdown/RackAwareness.md| 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/654372db/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
index 6a52f51..89946d1 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -91,12 +91,12 @@ sys.argv.pop(0) 
 # discard name
 netmask = '255.255.255.0'# set netmask 
to what's being used in your environment.  The example uses a /24
 
 for ip in sys.argv:  # loop over 
list of datanode IP's
-address = '{0}/{1}'.format(ip, netmask)  # format address 
string so it looks like 'ip/netmask' to make netaddr work
-try:
-   network_address = netaddr.IPNetwork(address).network # calculate and 
print network address
-   print "/{0}".format(network_address)
-except:
-   print "/rack-unknown"# print catch-all 
value if unable to calculate network address
+address = '{0}/{1}'.format(ip, netmask)  # format 
address string so it looks like 'ip/netmask' to make netaddr work
+try:
+network_address = netaddr.IPNetwork(address).network # calculate 
and print network address
+print "/{0}".format(network_address)
+except:
+print "/rack-unknown"# print 
catch-all value if unable to calculate network address
 ```
 
 bash Example


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14318. Remove non-existent setfattr command option from FileSystemShell.md. Contributed by Doris Gu.

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8.1 562af708c -> 15c6d6847


HADOOP-14318. Remove non-existent setfattr command option from 
FileSystemShell.md. Contributed by Doris Gu.

(cherry picked from commit 84a8848aaecd56b4cc85185924c2e33674165e01)
(cherry picked from commit a17bd7089263511b46918948843d05bb18750b1f)
(cherry picked from commit c8bcc96500b57db4cf34c539ea8067ed00eb6535)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15c6d684
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15c6d684
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15c6d684

Branch: refs/heads/branch-2.8.1
Commit: 15c6d684755d8e0f7045336c88d4c1a92c5c579f
Parents: 562af70
Author: Akira Ajisaka 
Authored: Tue Apr 18 16:43:50 2017 +0900
Committer: Akira Ajisaka 
Committed: Tue Apr 18 16:46:39 2017 +0900

--
 .../hadoop-common/src/site/markdown/FileSystemShell.md  | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/15c6d684/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index e970a69..03ec0d8 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -626,7 +626,6 @@ Sets an extended attribute name and value for a file or 
directory.
 
 Options:
 
-* -b: Remove all but the base ACL entries. The entries for user, group and 
others are retained for compatibility with permission bits.
 * -n name: The extended attribute name.
 * -v value: The extended attribute value. There are three different encoding 
methods for the value. If the argument is enclosed in double quotes, then the 
value is the string inside the quotes. If the argument is prefixed with 0x or 
0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 
0S, then it is taken as a base64 encoding.
 * -x name: Remove the extended attribute.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14318. Remove non-existent setfattr command option from FileSystemShell.md. Contributed by Doris Gu.

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 8b3b3aac1 -> c8bcc9650


HADOOP-14318. Remove non-existent setfattr command option from 
FileSystemShell.md. Contributed by Doris Gu.

(cherry picked from commit 84a8848aaecd56b4cc85185924c2e33674165e01)
(cherry picked from commit a17bd7089263511b46918948843d05bb18750b1f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8bcc965
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8bcc965
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8bcc965

Branch: refs/heads/branch-2.8
Commit: c8bcc96500b57db4cf34c539ea8067ed00eb6535
Parents: 8b3b3aa
Author: Akira Ajisaka 
Authored: Tue Apr 18 16:43:50 2017 +0900
Committer: Akira Ajisaka 
Committed: Tue Apr 18 16:46:01 2017 +0900

--
 .../hadoop-common/src/site/markdown/FileSystemShell.md  | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8bcc965/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index e970a69..03ec0d8 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -626,7 +626,6 @@ Sets an extended attribute name and value for a file or 
directory.
 
 Options:
 
-* -b: Remove all but the base ACL entries. The entries for user, group and 
others are retained for compatibility with permission bits.
 * -n name: The extended attribute name.
 * -v value: The extended attribute value. There are three different encoding 
methods for the value. If the argument is enclosed in double quotes, then the 
value is the string inside the quotes. If the argument is prefixed with 0x or 
0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 
0S, then it is taken as a base64 encoding.
 * -x name: Remove the extended attribute.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14318. Remove non-existent setfattr command option from FileSystemShell.md. Contributed by Doris Gu.

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c3840bde5 -> a17bd7089


HADOOP-14318. Remove non-existent setfattr command option from 
FileSystemShell.md. Contributed by Doris Gu.

(cherry picked from commit 84a8848aaecd56b4cc85185924c2e33674165e01)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a17bd708
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a17bd708
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a17bd708

Branch: refs/heads/branch-2
Commit: a17bd7089263511b46918948843d05bb18750b1f
Parents: c3840bd
Author: Akira Ajisaka 
Authored: Tue Apr 18 16:43:50 2017 +0900
Committer: Akira Ajisaka 
Committed: Tue Apr 18 16:45:14 2017 +0900

--
 .../hadoop-common/src/site/markdown/FileSystemShell.md  | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a17bd708/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index b1cf92f..5bfd57e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -626,7 +626,6 @@ Sets an extended attribute name and value for a file or 
directory.
 
 Options:
 
-* -b: Remove all but the base ACL entries. The entries for user, group and 
others are retained for compatibility with permission bits.
 * -n name: The extended attribute name.
 * -v value: The extended attribute value. There are three different encoding 
methods for the value. If the argument is enclosed in double quotes, then the 
value is the string inside the quotes. If the argument is prefixed with 0x or 
0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 
0S, then it is taken as a base64 encoding.
 * -x name: Remove the extended attribute.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: HADOOP-14318. Remove non-existent setfattr command option from FileSystemShell.md. Contributed by Doris Gu.

[aajisaka]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 8dfcd95d5 -> 84a8848aa


HADOOP-14318. Remove non-existent setfattr command option from 
FileSystemShell.md. Contributed by Doris Gu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84a8848a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84a8848a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84a8848a

Branch: refs/heads/trunk
Commit: 84a8848aaecd56b4cc85185924c2e33674165e01
Parents: 8dfcd95
Author: Akira Ajisaka 
Authored: Tue Apr 18 16:43:50 2017 +0900
Committer: Akira Ajisaka 
Committed: Tue Apr 18 16:43:50 2017 +0900

--
 .../hadoop-common/src/site/markdown/FileSystemShell.md  | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/84a8848a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 5db96eb..6614b31 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -626,7 +626,6 @@ Sets an extended attribute name and value for a file or 
directory.
 
 Options:
 
-* -b: Remove all but the base ACL entries. The entries for user, group and 
others are retained for compatibility with permission bits.
 * -n name: The extended attribute name.
 * -v value: The extended attribute value. There are three different encoding 
methods for the value. If the argument is enclosed in double quotes, then the 
value is the string inside the quotes. If the argument is prefixed with 0x or 
0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 
0S, then it is taken as a base64 encoding.
 * -x name: Remove the extended attribute.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[30/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
new file mode 100644
index 000..7e8cf5b
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiConstants.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.util;
+
+public interface RestApiConstants {
+  String CONTEXT_ROOT = "/services/v1";
+  String APPLICATIONS_API_RESOURCE_PATH = "/applications";
+  String CONTAINERS_API_RESOURCE_PATH = "/containers";
+  String SLIDER_APPMASTER_COMPONENT_NAME = "slider-appmaster";
+  String SLIDER_CONFIG_SCHEMA = "http://example.org/specification/v2.0.0;;
+  String METAINFO_SCHEMA_VERSION = "2.1";
+  String COMPONENT_TYPE_YARN_DOCKER = "yarn_docker";
+
+  String DEFAULT_START_CMD = "/bootstrap/privileged-centos6-sshd";
+  String DEFAULT_COMPONENT_NAME = "DEFAULT";
+  String DEFAULT_IMAGE = "centos:centos6";
+  String DEFAULT_NETWORK = "bridge";
+  String DEFAULT_COMMAND_PATH = "/usr/bin/docker";
+  String DEFAULT_USE_NETWORK_SCRIPT = "yes";
+
+  String PLACEHOLDER_APP_NAME = "${APP_NAME}";
+  String PLACEHOLDER_APP_COMPONENT_NAME = "${APP_COMPONENT_NAME}";
+  String PLACEHOLDER_COMPONENT_ID = "${COMPONENT_ID}";
+
+  String PROPERTY_REST_SERVICE_HOST = "REST_SERVICE_HOST";
+  String PROPERTY_REST_SERVICE_PORT = "REST_SERVICE_PORT";
+  String PROPERTY_APP_LIFETIME = "docker.lifetime";
+  String PROPERTY_APP_RUNAS_USER = "APP_RUNAS_USER";
+  Long DEFAULT_UNLIMITED_LIFETIME = -1l;
+
+  Integer HTTP_STATUS_CODE_ACCEPTED = 202;
+  String ARTIFACT_TYPE_SLIDER_ZIP = "slider-zip";
+
+  Integer GET_APPLICATIONS_THREAD_POOL_SIZE = 200;
+
+  String PROPERTY_PYTHON_PATH = "python.path";
+  String PROPERTY_DNS_DEPENDENCY = "site.global.dns.dependency";
+
+  String COMMAND_ORDER_SUFFIX_START = "-START";
+  String COMMAND_ORDER_SUFFIX_STARTED = "-STARTED";
+  String EXPORT_GROUP_NAME = "QuickLinks";
+
+  Integer ERROR_CODE_APP_DOES_NOT_EXIST = 404001;
+  Integer ERROR_CODE_APP_IS_NOT_RUNNING = 404002;
+  Integer ERROR_CODE_APP_SUBMITTED_BUT_NOT_RUNNING_YET = 404003;
+  Integer ERROR_CODE_APP_NAME_INVALID = 404004;
+
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
new file mode 100644
index 000..0f6247d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/util/RestApiErrorMessages.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under 

[06/50] [abbrv] hadoop git commit: YARN-5944. Native services AM should remain up if RM is down. Contributed by Billie Rinaldi

[jianhe]

YARN-5944. Native services AM should remain up if RM is down. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/df5c55a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/df5c55a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/df5c55a3

Branch: refs/heads/yarn-native-services
Commit: df5c55a32e523a08b7f35a6d7c0c66246c3efe84
Parents: 62154ae
Author: Gour Saha 
Authored: Thu Dec 1 00:30:01 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../org/apache/slider/server/appmaster/SliderAppMaster.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/df5c55a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 34b6a7d..8c39343 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -705,6 +705,11 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 synchronized (appState) {
   int heartbeatInterval = HEARTBEAT_INTERVAL;
 
+  // configure AM to wait forever for RM
+  
getConfig().setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+  -1);
+  getConfig().unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
   // add the RM client -this brings the callbacks in
   asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, 
this);
   addService(asyncRMClient);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[18/50] [abbrv] hadoop git commit: YARN-5996. Native services AM kills app on AMRMClientAsync onError call. Contributed by Billie Rinaldi

[jianhe]

YARN-5996. Native services AM kills app on AMRMClientAsync onError call. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/602b1547
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/602b1547
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/602b1547

Branch: refs/heads/yarn-native-services
Commit: 602b15471a734c8c23a7454ccf764ceee589d426
Parents: ea0d63e
Author: Gour Saha 
Authored: Fri Dec 16 12:09:29 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../org/apache/slider/server/appmaster/SliderAppMaster.java   | 7 +--
 1 file changed, 1 insertion(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/602b1547/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 74dbc88..611a3e5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1937,12 +1937,7 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 
   @Override //AMRMClientAsync
   public void onError(Throwable e) {
-//callback says it's time to finish
-LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
-signalAMComplete(new ActionStopSlider("stop",
-EXIT_EXCEPTION_THROWN,
-FinalApplicationStatus.FAILED,
-"AMRMClientAsync.onError() received " + e));
+LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
   }
   
 /* === */


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[17/50] [abbrv] hadoop git commit: YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

[jianhe]

YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5712253c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5712253c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5712253c

Branch: refs/heads/yarn-native-services
Commit: 5712253c0313e0881e772485a5931f9c9000f057
Parents: 2aeeb6f
Author: Billie Rinaldi 
Authored: Tue Dec 13 10:16:09 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../slider/api/SliderClusterProtocol.java   |   3 -
 .../slider/api/proto/RestTypeMarshalling.java   |  36 --
 .../org/apache/slider/client/SliderClient.java  |  80 ---
 .../client/ipc/SliderClusterOperations.java |  28 +-
 .../slider/common/params/ActionClientArgs.java  |  31 +-
 .../apache/slider/providers/ProviderUtils.java  | 141 -
 .../providers/docker/DockerProviderService.java |   5 -
 .../server/appmaster/SliderAppMaster.java   |  18 +-
 .../rpc/SliderClusterProtocolPBImpl.java|  12 -
 .../rpc/SliderClusterProtocolProxy.java |  18 +-
 .../server/appmaster/rpc/SliderIPCService.java  |  54 +-
 .../slider/server/appmaster/web/WebAppApi.java  |  23 -
 .../server/appmaster/web/WebAppApiImpl.java |  37 +-
 .../AbstractSecurityStoreGenerator.java |  98 
 .../services/security/CertificateManager.java   | 495 -
 .../services/security/KeystoreGenerator.java|  64 ---
 .../server/services/security/SecurityStore.java |  66 ---
 .../security/SecurityStoreGenerator.java|  40 --
 .../server/services/security/SecurityUtils.java | 256 -
 .../services/security/SignCertResponse.java |  67 ---
 .../server/services/security/SignMessage.java   |  54 --
 .../services/security/StoresGenerator.java  |  68 ---
 .../services/security/TruststoreGenerator.java  |  62 ---
 .../src/main/proto/SliderClusterProtocol.proto  |   6 -
 .../security/TestCertificateManager.java| 540 ---
 .../TestMultiThreadedStoreGeneration.java   | 156 --
 26 files changed, 14 insertions(+), 2444 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5712253c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 33fce22..893e706 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -173,7 +173,4 @@ public interface SliderClusterProtocol extends 
VersionedProtocol {
   Messages.WrappedJsonProto 
getModelResolvedResources(Messages.EmptyPayloadProto request) throws 
IOException;
 
   Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto 
request) throws IOException;
-
-  Messages.GetCertificateStoreResponseProto 
getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
-  throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5712253c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 17fd965..ec35028 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -18,8 +18,6 @@
 
 package org.apache.slider.api.proto;
 
-import com.google.protobuf.ByteString;
-import 

[11/50] [abbrv] hadoop git commit: YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1ce97eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
new file mode 100644
index 000..37f0d23
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -0,0 +1,561 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.net.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.ARecord;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.DNSKEYRecord;
+import org.xbill.DNS.DNSSEC;
+import org.xbill.DNS.Flags;
+import org.xbill.DNS.Message;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.OPTRecord;
+import org.xbill.DNS.PTRRecord;
+import org.xbill.DNS.RRSIGRecord;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Rcode;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SRVRecord;
+import org.xbill.DNS.Section;
+import org.xbill.DNS.Type;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.security.KeyFactory;
+import java.security.PrivateKey;
+import java.security.spec.RSAPrivateKeySpec;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+import static 
org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_MASK;
+import static 
org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_SUBNET;
+
+/**
+ *
+ */
+public class TestRegistryDNS extends Assert {
+
+  private RegistryDNS registryDNS;
+  private RegistryUtils.ServiceRecordMarshal marshal;
+
+  private static final String APPLICATION_RECORD = "{\n"
+  + "  \"type\" : \"JSONServiceRecord\",\n"
+  + "  \"description\" : \"Slider Application Master\",\n"
+  + "  \"external\" : [ {\n"
+  + "\"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+  + "\"addressType\" : \"host/port\",\n"
+  + "\"protocolType\" : \"hadoop/IPC\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"host\" : \"192.168.1.5\",\n"
+  + "  \"port\" : \"1026\"\n"
+  + "} ]\n"
+  + "  }, {\n"
+  + "\"api\" : \"http://\",\n;
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"webui\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"http://192.168.1.5:1027\"\n;
+  + "} ]\n"
+  + "  }, {\n"
+  + "\"api\" : \"classpath:org.apache.slider.management\",\n"
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"REST\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"http://192.168.1.5:1027/ws/v1/slider/mgmt\"\n;
+  + "} ]\n"
+  + "  } ],\n"
+  + "  \"internal\" : [ {\n"
+  + "\"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"REST\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n;
+  + "} ]\n"
+  + "  }, {\n"
+  + "\"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"REST\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"https://192.168.1.5:35531/ws/v1/slider/agents\"\n;
+  + "

[01/50] [abbrv] hadoop git commit: YARN-5808. Add gc log options to the yarn daemon script when starting services-api. Contributed by Billie Rinaldi [Forced Update!]

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services 3bffdc15e -> ccd2b0b35 (forced update)


YARN-5808. Add gc log options to the yarn daemon script when starting 
services-api. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02d3fae7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02d3fae7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02d3fae7

Branch: refs/heads/yarn-native-services
Commit: 02d3fae7ee675690824fae020e8be27677f38c7e
Parents: 69b454d
Author: Gour Saha 
Authored: Thu Nov 10 11:35:02 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn | 16 
 hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 12 
 2 files changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/02d3fae7/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index e50fb0d..39ef8d2 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,7 @@ function hadoop_usage
   hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" "admin tools"
   hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
-  hadoop_add_subcommand "services-api" "run slider services api"
+  hadoop_add_subcommand "servicesapi" "run slider services api"
   hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager 
daemon"
   hadoop_add_subcommand "slider" "run a slider app"
   hadoop_add_subcommand "timelinereader" "run the timeline reader server"
@@ -142,20 +142,20 @@ function yarncmd_case
 scmadmin)
   HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
 ;;
-services-api)
+servicesapi)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 sharedcachemanager)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -164,15 +164,15 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
 slider)
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   HADOOP_CLASSNAME='org.apache.slider.Slider'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 timelinereader)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02d3fae7/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh 
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index d003adb..3828897 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -136,3 +136,15 @@
 # See ResourceManager for some examples
 #
 #export YARN_SHAREDCACHEMANAGER_OPTS=
+
+###
+# Services API specific parameters
+###
+# Specify the JVM options to be used when starting the services API.
+#
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#

[40/50] [abbrv] hadoop git commit: YARN-6205. Default lifetime for native services app is invalid. Contributed by Billie Rinaldi

[jianhe]

YARN-6205. Default lifetime for native services app is invalid. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c90af1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c90af1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c90af1e

Branch: refs/heads/yarn-native-services
Commit: 7c90af1ea6d8df314b28b1d43689e7625cc5d931
Parents: fa7705e
Author: Gour Saha 
Authored: Tue Feb 21 09:04:04 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../src/main/java/org/apache/slider/client/SliderClient.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c90af1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index e84809a..00e2b62 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2070,7 +2070,7 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 amLauncher.setKeepContainersOverRestarts(true);
 // set lifetime in submission context;
 Map appTimeout = new HashMap<>();
-if (lifetime >= 0) {
+if (lifetime > 0) {
   appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
 }
 amLauncher.submissionContext.setApplicationTimeouts(appTimeout);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[27/50] [abbrv] hadoop git commit: YARN-6192. Move yarn-native-service API records into hadoop-yarn-slider-core module. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d557d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
deleted file mode 100644
index 5df00a0..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/PlacementPolicy.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.services.resource;
-
-import io.swagger.annotations.ApiModel;
-import io.swagger.annotations.ApiModelProperty;
-
-import java.io.Serializable;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-/**
- * Placement policy of an instance of an application. This feature is in the
- * works in YARN-4902.
- **/
-
-@ApiModel(description = "Placement policy of an instance of an application. 
This feature is in the works in YARN-4902.")
-@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
-public class PlacementPolicy implements Serializable {
-  private static final long serialVersionUID = 4341110649551172231L;
-
-  private String label = null;
-
-  /**
-   * Assigns an app to a named partition of the cluster where the application
-   * desires to run (optional). If not specified all apps are submitted to a
-   * default label of the app owner. One or more labels can be setup for each
-   * application owner account with required constraints like no-preemption,
-   * sla-9, preemption-ok, etc.
-   **/
-  public PlacementPolicy label(String label) {
-this.label = label;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", value = "Assigns an app to a named 
partition of the cluster where the application desires to run (optional). If 
not specified all apps are submitted to a default label of the app owner. One 
or more labels can be setup for each application owner account with required 
constraints like no-preemption, sla-9, preemption-ok, etc.")
-  @JsonProperty("label")
-  public String getLabel() {
-return label;
-  }
-
-  public void setLabel(String label) {
-this.label = label;
-  }
-
-  @Override
-  public boolean equals(java.lang.Object o) {
-if (this == o) {
-  return true;
-}
-if (o == null || getClass() != o.getClass()) {
-  return false;
-}
-PlacementPolicy placementPolicy = (PlacementPolicy) o;
-return Objects.equals(this.label, placementPolicy.label);
-  }
-
-  @Override
-  public int hashCode() {
-return Objects.hash(label);
-  }
-
-  @Override
-  public String toString() {
-StringBuilder sb = new StringBuilder();
-sb.append("class PlacementPolicy {\n");
-
-sb.append("label: ").append(toIndentedString(label)).append("\n");
-sb.append("}");
-return sb.toString();
-  }
-
-  /**
-   * Convert the given object to string with each line indented by 4 spaces
-   * (except the first line).
-   */
-  private String toIndentedString(java.lang.Object o) {
-if (o == null) {
-  return "null";
-}
-return o.toString().replace("\n", "\n");
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d557d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ReadinessCheck.java
 

[09/50] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

YARN-5961. Generate native services protobuf classes during build. Contributed 
by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2aeeb6f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2aeeb6f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2aeeb6f3

Branch: refs/heads/yarn-native-services
Commit: 2aeeb6f319d0f1a74d2e41e0ec32f4c57c7cacab
Parents: 40d2c28
Author: Jian He 
Authored: Sat Dec 3 13:27:04 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../hadoop-yarn-slider-core/pom.xml |65 +-
 .../org/apache/slider/api/proto/Messages.java   | 34473 -
 .../slider/api/proto/SliderClusterAPI.java  |  2293 --
 3 files changed, 27 insertions(+), 36804 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aeeb6f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 10cf6b1..7453d12 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -37,6 +37,33 @@
 
 
   
+org.apache.hadoop
+hadoop-maven-plugins
+
+  
+compile-protoc
+
+  protoc
+
+
+  ${protobuf.version}
+  ${protoc.path}
+  
+${basedir}/src/main/proto
+  
+  
+${basedir}/src/main/proto
+
+  SliderClusterMessages.proto
+  SliderClusterProtocol.proto
+
+  
+
+  
+
+  
+
+  
 org.apache.maven.plugins
 maven-jar-plugin
 
@@ -85,9 +112,6 @@
   
 **/*.json
 src/main/resources/webapps/slideram/.keep
-
-
src/main/java/org/apache/slider/api/proto/Messages.java
-
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
   
 
   
@@ -362,41 +386,6 @@
 
   
 
-
-  compile-protobuf
-  
-
-  
-org.apache.hadoop
-hadoop-maven-plugins
-
-  
-compile-protoc
-generate-sources
-
-  protoc
-
-
-  ${protobuf.version}
-  protoc
-  
-${basedir}/src/main/proto
-  
-  
-${basedir}/src/main/proto
-
-  SliderClusterMessages.proto
-  SliderClusterProtocol.proto
-
-  
-  ${basedir}/src/main/java
-
-  
-
-  
-
-  
-
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[45/50] [abbrv] hadoop git commit: YARN-6400. Remove some unneeded code after YARN-6255. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
deleted file mode 100644
index 68bd8a2..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveContainersRefresher.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.resources;
-
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Refresh the container list.
- */
-public class LiveContainersRefresher implements ResourceRefresher> {
-
-  private final StateAccessForProviders state;
-
-  public LiveContainersRefresher(StateAccessForProviders state) {
-this.state = state;
-  }
-
-  @Override
-  public Map refresh() throws
-  Exception {
-List containerList = state.cloneOwnedContainerList();
-
-Map map = new HashMap<>();
-for (RoleInstance instance : containerList) {
-  ContainerInformation serialized = instance.serialize();
-  map.put(serialized.containerId, serialized);
-}
-return map;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
deleted file mode 100644
index aeb7a11..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.web.rest.application.resources;
-
-import org.apache.slider.api.types.NodeInformationList;
-import 

[24/50] [abbrv] hadoop git commit: YARN-6115. Few additional paths in Slider client still uses get all Applications without tags/states filter. Contributed by Gour Saha

[jianhe]

YARN-6115. Few additional paths in Slider client still uses get all 
Applications without tags/states filter. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f28a2ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f28a2ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f28a2ac

Branch: refs/heads/yarn-native-services
Commit: 5f28a2accac0e9c953276a46a2b5f2449bc58493
Parents: 4faaf9f
Author: Billie Rinaldi 
Authored: Wed Jan 25 14:20:58 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../org/apache/slider/client/SliderClient.java  | 43 +++--
 .../slider/core/registry/YarnAppListClient.java | 51 +---
 2 files changed, 73 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f28a2ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1a959d6..032eb6e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -142,7 +142,6 @@ import org.apache.slider.core.launch.ClasspathConstructor;
 import org.apache.slider.core.launch.CredentialUtils;
 import org.apache.slider.core.launch.JavaCommandLineBuilder;
 import org.apache.slider.core.launch.LaunchedApplication;
-import org.apache.slider.core.launch.RunningApplication;
 import org.apache.slider.core.launch.SerializedApplicationReport;
 import org.apache.slider.core.main.RunService;
 import org.apache.slider.core.persist.AppDefinitionPersister;
@@ -1535,7 +1534,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 
   public void updateLifetime(String appName, long lifetime)
   throws YarnException, IOException {
-ApplicationReport report = findInstance(appName);
+EnumSet appStates = EnumSet.range(
+YarnApplicationState.NEW, YarnApplicationState.RUNNING);
+ApplicationReport report = findInstance(appName, appStates);
 if (report == null) {
   throw new YarnException("Application not found for " + appName);
 }
@@ -2672,15 +2673,17 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   }
 
   /**
-   * List Slider instances belonging to a specific user. This will include
-   * failed and killed instances; there may be duplicates
+   * List Slider instances belonging to a specific user with a specific app
+   * name and within a set of app states.
* @param user user: "" means all users, null means "default"
+   * @param appName name of the application set as a tag
+   * @param appStates a set of states the applications should be in
* @return a possibly empty list of Slider AMs
*/
-
-  public List listSliderInstances(String user)
-throws YarnException, IOException {
-return yarnAppListClient.listInstances(user);
+  public List listSliderInstances(String user,
+  String appName, EnumSet appStates)
+  throws YarnException, IOException {
+return yarnAppListClient.listInstances(user, appName, appStates);
   }
 
   /**
@@ -2806,7 +2809,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 }
 
 // and those the RM knows about
-List instances = listSliderInstances(null);
+EnumSet appStates = EnumSet.range(min, max);
+List instances = listSliderInstances(null, clustername,
+appStates);
 sortApplicationsByMostRecent(instances);
 Map reportMap =
 buildApplicationReportMap(instances, min, max);
@@ -3053,7 +3058,7 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   }
 
   /**
-   * Find an instance of an application belonging to the current user
+   * Find an instance of an application belonging to the current user.
* @param appname application name
* @return the app report or null if none is found
* @throws YarnException YARN issues
@@ -3061,14 +3066,22 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
*/

[05/50] [abbrv] hadoop git commit: YARN-5958. Fix ASF license warnings for slider core module. Contributed by Billie Rinaldi

[jianhe]

YARN-5958. Fix ASF license warnings for slider core module. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40d2c28e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40d2c28e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40d2c28e

Branch: refs/heads/yarn-native-services
Commit: 40d2c28eb2ece27795270868fed9eea5dc5e21eb
Parents: df5c55a
Author: Gour Saha 
Authored: Thu Dec 1 17:45:44 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../hadoop-yarn-slider-core/pom.xml | 61 +---
 .../src/license/THIRD-PARTY.properties  | 33 ---
 2 files changed, 14 insertions(+), 80 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/40d2c28e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 66e9ee9..10cf6b1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -78,6 +78,20 @@
 
   
 
+  
+org.apache.rat
+apache-rat-plugin
+
+  
+**/*.json
+src/main/resources/webapps/slideram/.keep
+
+
src/main/java/org/apache/slider/api/proto/Messages.java
+
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+  
+
+  
+
 
   
   
@@ -384,53 +398,6 @@
   
 
 
-
-
-  rat
-  
-
-
-  
-org.apache.rat
-apache-rat-plugin
-
-  
-check-licenses
-
-  check
-
-  
-
-
-  
-**/*.json
-src/test/python/agent.ini
-src/test/python/version
-**/THIRD-PARTY.properties
-src/main/resources/webapps/slideram/.keep
-src/main/resources/webapps/slideragent/.keep
-
src/main/resources/webapps/static/yarn.dt.plugins.js
-
-
src/main/resources/webapps/static/dt-1.9.4/**
-
-
src/main/resources/webapps/static/jquery/jquery-1.8.2.min.js
-
-
src/main/resources/webapps/static/jquery/jquery-ui-1.9.1.custom.min.js
-
src/main/resources/webapps/static/jquery/themes-1.9.1/base/jquery-ui.css
-
-
src/main/resources/webapps/static/jt/jquery.jstree.js
-
-
src/main/java/org/apache/slider/api/proto/Messages.java
-
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
-
src/test/app_packages/test_am_config/resources/test.template
-
src/test/app_packages/test_am_config/test_archive/testfile
-  
-
-  
-
-  
-
-
   
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/40d2c28e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
deleted file mode 100644
index 1abd56e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/license/THIRD-PARTY.properties
+++ /dev/null
@@ -1,33 +0,0 @@
-# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
-#---
-# Already used licenses in project :
-# - Apache License
-# - BSD
-# - CDDL + GPLv2 with classpath exception
-# - CDDL 1.1
-# - CDDL License
-# - CDDL+GPL
-# - Common Public License Version 1.0
-# - Eclipse Public License - Version 1.0
-# - GNU Lesser General Public License (LGPL), Version 2.1
-# - GNU Lesser General Public License, Version 2.1
-# - GPL2 w/ CPE
-# - MIT License
-# - MPL 1.1
-# - New BSD License
-# - Public 

[44/50] [abbrv] hadoop git commit: YARN-6186 Handle InvalidResourceRequestException in native services AM onError

[jianhe]

YARN-6186 Handle InvalidResourceRequestException in native services AM onError


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea37ea2d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea37ea2d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea37ea2d

Branch: refs/heads/yarn-native-services
Commit: ea37ea2d8cb473db954021d40dd21f57ed1074a3
Parents: d33f9c6
Author: Gour Saha 
Authored: Wed Feb 15 12:14:51 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../slider/server/appmaster/SliderAppMaster.java  | 18 +-
 .../server/appmaster/actions/ActionHalt.java  |  6 ++
 2 files changed, 23 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea37ea2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 611a3e5..a9a6b6b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import 
org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
+import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -115,6 +116,7 @@ import org.apache.slider.providers.ProviderService;
 import org.apache.slider.providers.SliderProviderFactory;
 import org.apache.slider.providers.slideram.SliderAMClientProvider;
 import org.apache.slider.providers.slideram.SliderAMProviderService;
+import org.apache.slider.server.appmaster.actions.ActionHalt;
 import 
org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
 import org.apache.slider.server.appmaster.actions.ActionStopSlider;
 import org.apache.slider.server.appmaster.actions.ActionUpgradeContainers;
@@ -1937,7 +1939,21 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 
   @Override //AMRMClientAsync
   public void onError(Throwable e) {
-LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
+if (e instanceof InvalidResourceRequestException) {
+  // stop the cluster
+  LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
+  signalAMComplete(new ActionStopSlider("stop", EXIT_EXCEPTION_THROWN,
+  FinalApplicationStatus.FAILED,
+  "AMRMClientAsync.onError() received " + e));
+} else if (e instanceof InvalidApplicationMasterRequestException) {
+  // halt the AM
+  LOG_YARN.error("AMRMClientAsync.onError() received {}", e, e);
+  queue(new ActionHalt(EXIT_EXCEPTION_THROWN,
+  "AMRMClientAsync.onError() received " + e));
+} else {
+  // ignore and log
+  LOG_YARN.info("Ignoring AMRMClientAsync.onError() received {}", e);
+}
   }
   
 /* === */

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea37ea2d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java
index e2ad559..ee1bb72 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionHalt.java

[32/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 6f54959..e891a27 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -18,12 +18,12 @@
 
 package org.apache.slider.server.appmaster.state;
 
-import com.codahale.metrics.Metric;
-import com.codahale.metrics.MetricRegistry;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -31,42 +31,35 @@ import 
org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterDescriptionKeys;
-import org.apache.slider.api.ClusterDescriptionOperations;
 import org.apache.slider.api.ClusterNode;
 import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ApplicationState;
+import org.apache.slider.api.resource.Component;
 import org.apache.slider.api.types.ApplicationLivenessInformation;
 import org.apache.slider.api.types.ComponentInformation;
 import org.apache.slider.api.types.RoleStatistics;
 import org.apache.slider.common.SliderExitCodes;
 import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.tools.ConfigHelper;
 import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
 import org.apache.slider.core.exceptions.BadClusterStateException;
 import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.ErrorStrings;
 import org.apache.slider.core.exceptions.NoSuchNodeException;
 import org.apache.slider.core.exceptions.SliderInternalStateException;
 import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.persist.AggregateConfSerDeser;
-import org.apache.slider.core.persist.ConfTreeSerDeser;
 import org.apache.slider.providers.PlacementPolicy;
 import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.server.appmaster.management.LongGauge;
 import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.management.MetricsConstants;
+import org.apache.slider.server.appmaster.metrics.SliderMetrics;
 import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation;
 import org.apache.slider.server.appmaster.operations.ContainerRequestOperation;
@@ -77,7 +70,6 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -89,12 +81,10 @@ import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static org.apache.hadoop.metrics2.lib.Interns.info;
 import static org.apache.slider.api.ResourceKeys.*;

[46/50] [abbrv] hadoop git commit: YARN-6400. Remove some unneeded code after YARN-6255. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
deleted file mode 100644
index aad2757..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/conf/TemplateInputPropertiesValidator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.core.conf;
-
-import org.apache.slider.core.exceptions.BadConfigException;
-
-/**
- *
- */
-public class TemplateInputPropertiesValidator
-extends AbstractInputPropertiesValidator {
-
-  void validatePropertyNamePrefix(String key) throws BadConfigException {
-if (key.startsWith("yarn.")) {
-  throw new BadConfigException(
-  "argument %s has 'yarn.' prefix - this is not allowed in templates", 
key);
-}
-  }
-
-  @Override
-  void validateGlobalProperties(ConfTreeOperations props) {
-// do nothing
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index aefc0de..4182459 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -304,30 +304,6 @@ public abstract class AbstractLauncher extends Configured {
 }
   }
 
-  /**
-   * Extract the value for option
-   * {@code yarn.resourcemanager.am.retry-count-window-ms}
-   * and set it on the ApplicationSubmissionContext. Use the default value
-   * if option is not set.
-   *
-   * @param submissionContext
-   * @param map
-   */
-  public void extractAmRetryCount(ApplicationSubmissionContext 
submissionContext,
-  Map map) {
-
-if (map != null) {
-  MapOperations options = new MapOperations("", map);
-  long amRetryCountWindow = options.getOptionLong(ResourceKeys
-  .YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
-  ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS);
-  log.info("Setting {} to {}",
-  ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
-  amRetryCountWindow);
-  submissionContext.setAttemptFailuresValidityInterval(amRetryCountWindow);
-}
-  }
-
   public void extractLogAggregationContext(Map map) {
 if (map != null) {
   String logPatternSepStr = "\\|";
@@ -453,24 +429,6 @@ public abstract class AbstractLauncher extends Configured {
 env.putAll(map);
   }
 
-  /**
-   * Important: the configuration must already be fully resolved 
-   * in order to pick up global options
-   * Copy env vars into the launch context.
-   */
-  public boolean copyEnvVars(MapOperations options) {
-if (options == null) {
-  return false;
-}
-for (Map.Entry entry : options.entrySet()) {
-  String key = 

[26/50] [abbrv] hadoop git commit: YARN-6192. Move yarn-native-service API records into hadoop-yarn-slider-core module. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d557d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
new file mode 100644
index 000..c5dc627
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Container.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Date;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * An instance of a running application container.
+ **/
+
+@ApiModel(description = "An instance of a running application container")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class Container extends BaseResource {
+  private static final long serialVersionUID = -8955788064529288L;
+
+  private String id = null;
+  private Date launchTime = null;
+  private String ip = null;
+  private String hostname = null;
+  private String bareHost = null;
+  private ContainerState state = null;
+  private String componentName = null;
+  private Resource resource = null;
+  private Artifact artifact = null;
+  private Boolean privilegedContainer = null;
+
+  /**
+   * Unique container id of a running application, e.g.
+   * container_e3751_1458061340047_0008_01_02.
+   **/
+  public Container id(String id) {
+this.id = id;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Unique container id of a 
running application, e.g. container_e3751_1458061340047_0008_01_02.")
+  @JsonProperty("id")
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  /**
+   * The time when the container was created, e.g. 2016-03-16T01:01:49.000Z.
+   * This will most likely be different from cluster launch time.
+   **/
+  public Container launchTime(Date launchTime) {
+this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "The time when the container was 
created, e.g. 2016-03-16T01:01:49.000Z. This will most likely be different from 
cluster launch time.")
+  @JsonProperty("launch_time")
+  public Date getLaunchTime() {
+return launchTime == null ? null : (Date) launchTime.clone();
+  }
+
+  @XmlElement(name = "launch_time")
+  public void setLaunchTime(Date launchTime) {
+this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
+  }
+
+  /**
+   * IP address of a running container, e.g. 172.31.42.141. The IP address and
+   * hostname attribute values are dependent on the cluster/docker network 
setup
+   * as per YARN-4007.
+   **/
+  public Container ip(String ip) {
+this.ip = ip;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "IP address of a running 
container, e.g. 172.31.42.141. The IP address and hostname attribute values are 
dependent on the cluster/docker network setup as per YARN-4007.")
+  @JsonProperty("ip")
+  public String getIp() {
+return ip;
+  }
+
+  public void setIp(String ip) {
+this.ip = ip;
+  }
+
+  /**
+   * Fully qualified hostname of a running container, e.g.
+   * ctr-e3751-1458061340047-0008-01-02.examplestg.site. The IP address and
+   * hostname 

[07/50] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aeeb6f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
deleted file mode 100644
index 081b7fa..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+++ /dev/null
@@ -1,2293 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: SliderClusterProtocol.proto
-
-package org.apache.slider.api.proto;
-
-public final class SliderClusterAPI {
-  private SliderClusterAPI() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  /**
-   * Protobuf service {@code org.apache.slider.api.SliderClusterProtocolPB}
-   *
-   * 
-   **
-   * Protocol used from between Slider Client and AM
-   * 
-   */
-  public static abstract class SliderClusterProtocolPB
-  implements com.google.protobuf.Service {
-protected SliderClusterProtocolPB() {}
-
-public interface Interface {
-  /**
-   * rpc stopCluster(.org.apache.slider.api.StopClusterRequestProto) 
returns (.org.apache.slider.api.StopClusterResponseProto);
-   */
-  public abstract void stopCluster(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
upgradeContainers(.org.apache.slider.api.UpgradeContainersRequestProto) returns 
(.org.apache.slider.api.UpgradeContainersResponseProto);
-   *
-   * 
-   **
-   * Upgrade containers 
-   * 
-   */
-  public abstract void upgradeContainers(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto 
request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc flexCluster(.org.apache.slider.api.FlexClusterRequestProto) 
returns (.org.apache.slider.api.FlexClusterResponseProto);
-   *
-   * 
-   **
-   * Flex the cluster. 
-   * 
-   */
-  public abstract void flexCluster(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
getJSONClusterStatus(.org.apache.slider.api.GetJSONClusterStatusRequestProto) 
returns (.org.apache.slider.api.GetJSONClusterStatusResponseProto);
-   *
-   * 
-   **
-   * Get the current cluster status
-   * 
-   */
-  public abstract void getJSONClusterStatus(
-  com.google.protobuf.RpcController controller,
-  
org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
getInstanceDefinition(.org.apache.slider.api.GetInstanceDefinitionRequestProto) 
returns (.org.apache.slider.api.GetInstanceDefinitionResponseProto);
-   *
-   * 
-   **
-   * Get the instance definition
-   * 
-   */
-  public abstract void getInstanceDefinition(
-  com.google.protobuf.RpcController controller,
-  
org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
listNodeUUIDsByRole(.org.apache.slider.api.ListNodeUUIDsByRoleRequestProto) 
returns (.org.apache.slider.api.ListNodeUUIDsByRoleResponseProto);
-   *
-   * 
-   **
-   * List all running nodes in a role
-   * 
-   */
-  public abstract void listNodeUUIDsByRole(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto 
request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc getNode(.org.apache.slider.api.GetNodeRequestProto) returns 
(.org.apache.slider.api.GetNodeResponseProto);
-   *
-   * 
-   **
-   * Get the details on a node
-   * 
-   */
-  public abstract void getNode(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 

[48/50] [abbrv] hadoop git commit: YARN-6400. Remove some unneeded code after YARN-6255. Contributed by Jian He

[jianhe]

YARN-6400. Remove some unneeded code after YARN-6255. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db9378fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db9378fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db9378fd

Branch: refs/heads/yarn-native-services
Commit: db9378fde2bd514f4c3dff79c27a19a95f5537fa
Parents: 538658d
Author: Jian He 
Authored: Thu Mar 30 16:07:18 2017 +0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:20 2017 +0800

--
 .../apache/slider/api/ClusterDescription.java   | 795 ---
 .../slider/api/ClusterDescriptionKeys.java  |  25 -
 .../api/ClusterDescriptionOperations.java   |  93 ---
 .../apache/slider/api/SliderApplicationApi.java | 159 
 .../slider/api/SliderClusterProtocol.java   |  14 -
 .../types/ApplicationLivenessInformation.java   |   3 -
 .../slider/api/types/RestTypeMarshalling.java   |  27 -
 .../org/apache/slider/client/SliderClient.java  | 437 +++---
 .../client/ipc/SliderApplicationIpcClient.java  | 234 --
 .../client/ipc/SliderClusterOperations.java | 127 +--
 .../slider/client/rest/RestClientFactory.java   |  89 ---
 .../rest/SliderApplicationApiRestClient.java| 289 ---
 .../AbstractClusterBuildingActionArgs.java  |  11 -
 .../slider/common/tools/CoreFileSystem.java |  74 --
 .../apache/slider/common/tools/SliderUtils.java | 116 ---
 .../slider/core/buildutils/InstanceBuilder.java | 520 
 .../slider/core/buildutils/InstanceIO.java  |  83 --
 .../conf/AbstractInputPropertiesValidator.java  |  49 --
 .../apache/slider/core/conf/AggregateConf.java  | 198 -
 .../org/apache/slider/core/conf/ConfTree.java   | 101 ---
 .../slider/core/conf/ConfTreeOperations.java| 527 
 .../core/conf/InputPropertiesValidator.java |  27 -
 .../conf/ResourcesInputPropertiesValidator.java |  41 -
 .../conf/TemplateInputPropertiesValidator.java  |  38 -
 .../slider/core/launch/AbstractLauncher.java|  55 --
 .../core/persist/AggregateConfSerDeser.java |  55 --
 .../slider/core/persist/ConfPersister.java  | 286 ---
 .../slider/core/persist/ConfTreeSerDeser.java   |  54 --
 .../persist/LockAcquireFailedException.java |  40 -
 .../providers/AbstractClientProvider.java   | 167 +---
 .../slider/providers/ProviderService.java   |  11 -
 .../providers/docker/DockerClientProvider.java  |  18 -
 .../server/appmaster/SliderAppMaster.java   |  87 +-
 .../appmaster/actions/ActionFlexCluster.java|   1 -
 .../rpc/SliderClusterProtocolPBImpl.java|  69 --
 .../rpc/SliderClusterProtocolProxy.java |  63 --
 .../server/appmaster/rpc/SliderIPCService.java  |  68 --
 .../security/SecurityConfiguration.java | 237 +++---
 .../slider/server/appmaster/state/AppState.java |  63 --
 .../appmaster/web/rest/AMWebServices.java   |   9 +-
 .../resources/AggregateModelRefresher.java  |  43 -
 .../application/resources/AppconfRefresher.java |  52 --
 .../resources/LiveComponentsRefresher.java  |  39 -
 .../resources/LiveContainersRefresher.java  |  52 --
 .../resources/LiveNodesRefresher.java   |  41 -
 .../web/rest/management/ManagementResource.java |  94 ---
 .../management/resources/ActionsResource.java   |  22 -
 .../resources/AggregateConfResource.java|  90 ---
 .../management/resources/ComponentResource.java |  53 --
 .../management/resources/ConfTreeResource.java  |  69 --
 .../management/resources/ResourceFactory.java   |  47 --
 .../web/rest/publisher/PublisherResource.java   |   7 +-
 .../src/main/proto/SliderClusterProtocol.proto  |  28 -
 53 files changed, 239 insertions(+), 5758 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
deleted file mode 100644
index f8e5e7c..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ /dev/null
@@ -1,795 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- *  or more contributor license agreements.  See the NOTICE file
- *  distributed with this work for additional 

[13/50] [abbrv] hadoop git commit: YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron

[jianhe]

YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1ce97eb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1ce97eb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1ce97eb

Branch: refs/heads/yarn-native-services
Commit: a1ce97eb9dd99a99706bae1e9b4cba869eacfcf3
Parents: 18fb4a5
Author: Jian He 
Authored: Sun Jun 12 11:32:03 2016 -0700
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 hadoop-project/pom.xml  |8 +
 .../dev-support/findbugs-exclude.xml|   15 +
 .../hadoop-yarn/hadoop-yarn-registry/pom.xml|5 +
 .../registry/client/api/DNSOperations.java  |   60 +
 .../client/api/DNSOperationsFactory.java|   78 +
 .../registry/client/api/RegistryConstants.java  |  111 +-
 .../registry/client/impl/zk/CuratorService.java |  266 ++-
 .../registry/client/impl/zk/ListenerHandle.java |   25 +
 .../registry/client/impl/zk/PathListener.java   |   30 +
 .../types/yarn/YarnRegistryAttributes.java  |   16 +-
 .../dns/ApplicationServiceRecordProcessor.java  |  353 
 .../server/dns/BaseServiceRecordProcessor.java  |  469 ++
 .../dns/ContainerServiceRecordProcessor.java|  278 
 .../server/dns/RecordCreatorFactory.java|  275 
 .../hadoop/registry/server/dns/RegistryDNS.java | 1534 ++
 .../registry/server/dns/RegistryDNSServer.java  |  290 
 .../registry/server/dns/SecureableZone.java |  151 ++
 .../server/dns/ServiceRecordProcessor.java  |   53 +
 .../registry/server/dns/ZoneSelector.java   |   33 +
 .../registry/server/dns/package-info.java   |   26 +
 .../registry/server/dns/TestRegistryDNS.java|  561 +++
 .../server/dns/TestSecureRegistryDNS.java   |   44 +
 .../test/resources/0.17.172.in-addr.arpa.zone   |   36 +
 .../src/test/resources/test.private |   32 +
 24 files changed, 4661 insertions(+), 88 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1ce97eb/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index ba5a4ae..cd0a51d 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -88,6 +88,7 @@
 3.4.9
 2.12.0
 3.0.0
+2.1.7
 
 6.0.48
 4.0
@@ -1182,6 +1183,13 @@
   
 
   
+
+  
+dnsjava
+dnsjava
+${dnsjava.version}
+  
+
   
 org.skyscreamer
 jsonassert

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1ce97eb/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml 
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c090749..9600a93 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -579,4 +579,19 @@
 
   
 
+  
+
+
+
+  
+  
+
+
+
+  
+  
+
+
+
+  
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1ce97eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 25bfa2e..5eeea0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -80,6 +80,11 @@
   test
 
 
+
+  dnsjava
+  dnsjava
+
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1ce97eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
new file mode 100644
index 000..3abfb6c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The 

[47/50] [abbrv] hadoop git commit: YARN-6400. Remove some unneeded code after YARN-6255. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
deleted file mode 100644
index 4286596..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.client.rest;
-
-import com.sun.jersey.api.client.Client;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.client.ClientRegistryBinder;
-import org.apache.slider.api.SliderApplicationApi;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-
-import java.io.IOException;
-
-import static 
org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
-
-/**
- * Factory for the Rest client; hides the lookup and instantiation.
- * 
- * 
- */
-public class RestClientFactory {
-
-  private final ClientRegistryBinder binder;
-  private final Client jerseyClient;
-  private final String user, serviceclass, instance;
-
-  public RestClientFactory(RegistryOperations operations,
-  Client jerseyClient,
-  String user,
-  String serviceclass,
-  String instance) {
-this.jerseyClient = jerseyClient;
-this.user = user;
-this.serviceclass = serviceclass;
-this.instance = instance;
-binder = new ClientRegistryBinder(operations);
-  }
-
-  /**
-   * Locate the AM
-   * @return a resource to the AM
-   * @throws IOException any failure to resolve to the AM
-   */
-  private WebResource locateAppmaster() throws IOException {
-String restAPI = binder.lookupExternalRestAPI(user, serviceclass, instance,
-CustomRegistryConstants.AM_REST_BASE);
-return jerseyClient.resource(restAPI);
-  }
-
-  /**
-   * Locate the slider AM then instantiate a client instance against
-   * its Application API.
-   * @return the instance
-   * @throws IOException on any failure
-   */
-  public SliderApplicationApi createSliderAppApiClient() throws IOException {
-WebResource appmaster = locateAppmaster();
-return createSliderAppApiClient(appmaster);
-  }
-
-   /**
-   * Create a Slider application API client instance against
-   * its Application API.
-   * @param appmaster The AM to work against.
-   * @return the instance
-   * @throws IOException on any failure
-   */
-  public SliderApplicationApi createSliderAppApiClient(WebResource appmaster) {
-WebResource appResource = appmaster.path(SLIDER_PATH_APPLICATION);
-return new SliderApplicationApiRestClient(jerseyClient, appResource);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/db9378fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
deleted file mode 100644
index 4c376e0..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*

[35/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
deleted file mode 100644
index 9eb7d5c..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.core.persist;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.io.Files;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
-import org.apache.slider.common.params.Arguments;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.providers.agent.AgentKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Class to prepare and persist app and add-on definitions.
- *
- * In this case, the app definition and add-on definitions are auto-inferred 
from the user input rather than explicit
- * inclusion of application package in the config.
- *
- * Processing an app definition involves one or more of the following: - 
modify appConfig - package definition into a
- * temporary folder - upload to HDFS
- *
- * This class keeps track of all the required operations and allows them to be 
invoked by build operation
- */
-public class AppDefinitionPersister {
-  private static final Logger log =
-  LoggerFactory.getLogger(AppDefinitionPersister.class);
-
-  private final SliderFileSystem sliderFileSystem;
-  private List appDefinitions;
-
-  public AppDefinitionPersister(SliderFileSystem sliderFileSystem) {
-this.sliderFileSystem = sliderFileSystem;
-appDefinitions = new ArrayList<>();
-  }
-
-
-  /**
-   * Process the application package or folder by copying it to the cluster 
path
-   *
-   * @param appDefinition details of application package
-   *
-   * @throws BadConfigException
-   * @throws IOException
-   */
-  private void persistDefinitionPackageOrFolder(AppDefinition appDefinition)
-  throws BadConfigException, IOException {
-if (!appDefinition.appDefPkgOrFolder.canRead()) {
-  throw new BadConfigException("Pkg/Folder cannot be accessed - "
-   + 
appDefinition.appDefPkgOrFolder.getAbsolutePath());
-}
-
-File src = appDefinition.appDefPkgOrFolder;
-String targetName = appDefinition.pkgName;
-log.debug("Package name: " + targetName);
-if (appDefinition.appDefPkgOrFolder.isDirectory()) {
-  log.info("Processing app package/folder {} for {}",
-   appDefinition.appDefPkgOrFolder.getAbsolutePath(),
-   appDefinition.pkgName);
-  File tmpDir = Files.createTempDir();
-  File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + 
appDefinition.pkgName);
-  SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile);
-  src = zipFile;
-}
-
-sliderFileSystem.getFileSystem().copyFromLocalFile(
-false,
-false,
-new Path(src.toURI()),
-new Path(appDefinition.targetFolderInFs, 

[41/50] [abbrv] hadoop git commit: YARN-6173. Add artifact info and privileged container details to the container info in API GET response. Contributed by Gour Saha

[jianhe]

YARN-6173. Add artifact info and privileged container details to the container 
info in API GET response. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d33f9c6d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d33f9c6d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d33f9c6d

Branch: refs/heads/yarn-native-services
Commit: d33f9c6da3440673ae31571f46e410f1fdf864a5
Parents: ef10c96
Author: Billie Rinaldi 
Authored: Tue Feb 14 09:52:41 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../api/impl/ApplicationApiService.java | 18 +++
 .../yarn/services/resource/Container.java   | 53 
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  6 +++
 3 files changed, 67 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33f9c6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index b11da2c..7028caa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -1000,6 +1000,19 @@ public class ApplicationApiService implements 
ApplicationApi {
   resource.setCpus(jsonGetAsInt(componentRole, "yarn.vcores"));
   resource.setMemory(jsonGetAsString(componentRole, 
"yarn.memory"));
   container.setResource(resource);
+  Artifact artifact = new Artifact();
+  String dockerImageName = jsonGetAsString(componentRole,
+  "docker.image");
+  if (StringUtils.isNotEmpty(dockerImageName)) {
+artifact.setId(dockerImageName);
+artifact.setType(Artifact.TypeEnum.DOCKER);
+  } else {
+// Might have to handle tarballs here
+artifact.setType(null);
+  }
+  container.setArtifact(artifact);
+  container.setPrivilegedContainer(
+  jsonGetAsBoolean(componentRole, "docker.usePrivileged"));
   // TODO: add container property - for response only?
   app.addContainer(container);
 }
@@ -1057,6 +1070,11 @@ public class ApplicationApiService implements 
ApplicationApi {
 : object.get(key).isJsonNull() ? null : object.get(key).getAsInt();
   }
 
+  private Boolean jsonGetAsBoolean(JsonObject object, String key) {
+return object.get(key) == null ? null
+: object.get(key).isJsonNull() ? null : object.get(key).getAsBoolean();
+  }
+
   private JsonObject jsonGetAsObject(JsonObject object, String key) {
 return object.get(key) == null ? null : object.get(key).getAsJsonObject();
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d33f9c6d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
index f11c7b3..24aada7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Container.java
@@ -48,6 +48,8 @@ public class Container extends BaseResource {
   private ContainerState state = null;
   private String componentName = null;
   private Resource resource = null;
+  private Artifact artifact = null;
+  private Boolean privilegedContainer = null;
 
   /**
* Unique container id of a running application, e.g.
@@ -204,6 +206,42 @@ public class Container 

[39/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/821abcd2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/821abcd2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/821abcd2

Branch: refs/heads/yarn-native-services
Commit: 821abcd2747c93c5c1ec25816e85a201b2a3039a
Parents: 23d557d
Author: Jian He 
Authored: Sun Mar 26 21:42:14 2017 +0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../yarn/services/api/ApplicationApi.java   |2 -
 .../api/impl/ApplicationApiService.java | 1477 ++--
 .../yarn/services/utils/RestApiConstants.java   |   63 -
 .../services/utils/RestApiErrorMessages.java|   83 -
 .../services/webapp/ApplicationApiWebApp.java   |2 +-
 .../api/impl/TestApplicationApiService.java |   65 +-
 .../apache/slider/api/SliderApplicationApi.java |8 -
 .../slider/api/SliderClusterProtocol.java   |   14 +-
 .../apache/slider/api/resource/Application.java |   16 +-
 .../apache/slider/api/resource/Component.java   |   38 +-
 .../apache/slider/api/resource/ConfigFile.java  |   26 +-
 .../slider/api/resource/Configuration.java  |   30 +
 .../apache/slider/api/resource/Resource.java|2 +-
 .../org/apache/slider/client/SliderClient.java  | 2139 --
 .../apache/slider/client/SliderClientAPI.java   |   71 +-
 .../slider/client/SliderYarnClientImpl.java |  200 +-
 .../client/ipc/SliderApplicationIpcClient.java  |   11 -
 .../client/ipc/SliderClusterOperations.java |   59 +-
 .../rest/SliderApplicationApiRestClient.java|   23 -
 .../org/apache/slider/common/SliderKeys.java|   10 +-
 .../apache/slider/common/SliderXmlConfKeys.java |3 +
 .../common/params/AbstractActionArgs.java   |2 +-
 .../AbstractClusterBuildingActionArgs.java  |  185 +-
 .../slider/common/params/ActionBuildArgs.java   |   32 -
 .../slider/common/params/ActionCreateArgs.java  |   34 +-
 .../slider/common/params/ActionFlexArgs.java|   34 +-
 .../slider/common/params/ActionFreezeArgs.java  |4 +-
 .../slider/common/params/ActionThawArgs.java|4 +-
 .../slider/common/params/ActionUpgradeArgs.java |   59 +-
 .../apache/slider/common/params/Arguments.java  |1 +
 .../apache/slider/common/params/ClientArgs.java |   16 +-
 .../slider/common/params/SliderAMArgs.java  |2 +-
 .../slider/common/params/SliderActions.java |6 +-
 .../slider/common/tools/CoreFileSystem.java |1 +
 .../slider/common/tools/SliderFileSystem.java   |9 +
 .../apache/slider/common/tools/SliderUtils.java |   32 +-
 .../slider/core/launch/AppMasterLauncher.java   |  233 --
 .../slider/core/launch/LaunchedApplication.java |  108 -
 .../slider/core/launch/RunningApplication.java  |   76 -
 .../core/persist/AppDefinitionPersister.java|  263 ---
 .../slider/core/persist/JsonSerDeser.java   |6 +
 .../providers/AbstractClientProvider.java   |   15 +-
 .../providers/AbstractProviderService.java  |  438 
 .../apache/slider/providers/ProviderCore.java   |   12 -
 .../apache/slider/providers/ProviderRole.java   |   19 +-
 .../slider/providers/ProviderService.java   |  157 +-
 .../apache/slider/providers/ProviderUtils.java  |  929 +---
 .../providers/docker/DockerClientProvider.java  |   35 +-
 .../providers/docker/DockerProviderService.java |  423 +---
 .../slideram/SliderAMClientProvider.java|  305 ---
 .../slideram/SliderAMProviderService.java   |  185 --
 .../server/appmaster/RoleLaunchService.java |  122 +-
 .../server/appmaster/SliderAppMaster.java   |  540 ++---
 .../appmaster/actions/ActionFlexCluster.java|   14 +-
 .../actions/ActionRegisterServiceInstance.java  |   17 +-
 .../server/appmaster/metrics/SliderMetrics.java |   80 +
 .../ProviderNotifyingOperationHandler.java  |   63 -
 .../rpc/SliderClusterProtocolPBImpl.java|   22 +-
 .../rpc/SliderClusterProtocolProxy.java |   19 +-
 .../server/appmaster/rpc/SliderIPCService.java  |   55 +-
 .../security/SecurityConfiguration.java |1 +
 .../slider/server/appmaster/state/AppState.java | 1031 ++---
 .../appmaster/state/AppStateBindingInfo.java|   10 +-
 .../appmaster/state/ProviderAppState.java   |   48 +-
 .../server/appmaster/state/RoleHistory.java |   15 +-
 .../server/appmaster/state/RoleInstance.java|2 +
 .../server/appmaster/state/RoleStatus.java  |  324 +--
 .../state/StateAccessForProviders.java  |   61 +-
 .../server/appmaster/web/SliderAMWebApp.java|   15 +-
 .../slider/server/appmaster/web/WebAppApi.java  |7 -
 .../server/appmaster/web/WebAppApiImpl.java |   15 +-
 .../appmaster/web/rest/AMWebServices.java   |   33 +-
 .../server/appmaster/web/rest/RestPaths.java  

[43/50] [abbrv] hadoop git commit: YARN-6185. Apply SLIDER-1199 to yarn native services for blacklisting nodes. Contributed by Billie Rinaldi

[jianhe]

YARN-6185. Apply SLIDER-1199 to yarn native services for blacklisting nodes. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa7705ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa7705ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa7705ec

Branch: refs/heads/yarn-native-services
Commit: fa7705ec3eccbb4aaddf40b79bfe2b989fd2bb7a
Parents: ea37ea2
Author: Gour Saha 
Authored: Tue Feb 21 08:52:47 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../providers/AbstractProviderService.java  |  6 +++
 .../server/appmaster/SliderAppMaster.java   |  8 +++-
 .../appmaster/actions/ResetFailureWindow.java   | 14 +-
 .../operations/AsyncRMOperationHandler.java |  6 +++
 .../ProviderNotifyingOperationHandler.java  |  8 
 .../operations/RMOperationHandlerActions.java   | 18 +---
 .../operations/UpdateBlacklistOperation.java| 45 
 .../slider/server/appmaster/state/AppState.java | 14 ++
 .../server/appmaster/state/NodeInstance.java| 10 +
 .../server/appmaster/state/RoleHistory.java | 34 +++
 10 files changed, 155 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa7705ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 00fc606..41b26e9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -411,6 +411,12 @@ public abstract class AbstractProviderService
   }
 
   @Override
+  public void updateBlacklist(List blacklistAdditions,
+  List blacklistRemovals) {
+// no-op
+  }
+
+  @Override
   public void execute(List operations) {
 for (AbstractRMOperation operation : operations) {
   operation.execute(this);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa7705ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index a9a6b6b..f67ea58 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1755,7 +1755,7 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
*/
   private void scheduleFailureWindowResets(ConfTree resources) throws
   BadConfigException {
-ResetFailureWindow reset = new ResetFailureWindow();
+ResetFailureWindow reset = new ResetFailureWindow(rmOperationHandler);
 ConfTreeOperations ops = new ConfTreeOperations(resources);
 MapOperations globals = ops.getGlobalOptions();
 long seconds = globals.getTimeRange(ResourceKeys.CONTAINER_FAILURE_WINDOW,
@@ -1988,6 +1988,12 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 rmOperationHandler.cancelSingleRequest(request);
   }
 
+  @Override
+  public void updateBlacklist(List blacklistAdditions,
+  List blacklistRemovals) {
+rmOperationHandler.updateBlacklist(blacklistAdditions, blacklistRemovals);
+  }
+
 /* === */
 /* END */
 /* === */

[36/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index c6cc2d0..2bb224b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.service.Service;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.types.NodeInformationList;
 import org.apache.slider.api.types.SliderInstanceDescription;
 import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
@@ -61,15 +62,8 @@ import java.util.Map;
  * Stability: evolving
  */
 public interface SliderClientAPI extends Service {
-  /**
-   * Destroy a cluster. There's two race conditions here
-   * #1 the cluster is started between verifying that there are no live
-   * clusters of that name.
-   */
-  int actionDestroy(String clustername, ActionDestroyArgs destroyArgs)
-  throws YarnException, IOException;
 
-  int actionDestroy(String clustername) throws YarnException,
+  void actionDestroy(String clustername) throws YarnException,
   IOException;
 
   /**
@@ -88,18 +82,6 @@ public interface SliderClientAPI extends Service {
 throws SliderException;
 
   /**
-   * Build up the cluster specification/directory
-   *
-   * @param clustername cluster name
-   * @param buildInfo the arguments needed to build the cluster
-   * @throws YarnException Yarn problems
-   * @throws IOException other problems
-   * @throws BadCommandArgumentsException bad arguments.
-   */
-  int actionBuild(String clustername,
-  AbstractClusterBuildingActionArgs buildInfo) throws YarnException, 
IOException;
-
-  /**
* Upload keytab to a designated sub-directory of the user home directory
*
* @param installKeytabInfo the arguments needed to upload the keytab
@@ -188,24 +170,7 @@ public interface SliderClientAPI extends Service {
*/
   int actionUpgrade(String clustername,
   ActionUpgradeArgs buildInfo)
-  throws YarnException, IOException; 
-
-  /**
-   * Get the report of a this application
-   * @return the app report or null if it could not be found.
-   * @throws IOException
-   * @throws YarnException
-   */
-  ApplicationReport getApplicationReport()
-  throws IOException, YarnException;
-
-  /**
-   * Kill the submitted application via YARN
-   * @throws YarnException
-   * @throws IOException
-   */
-  boolean forceKillApplication(String reason)
-throws YarnException, IOException;
+  throws YarnException, IOException;
 
   /**
* Implement the list action: list all nodes
@@ -213,30 +178,8 @@ public interface SliderClientAPI extends Service {
*/
   int actionList(String clustername, ActionListArgs args) throws IOException, 
YarnException;
 
-  /**
-   * Enumerate slider instances for the current user, and the
-   * most recent app report, where available.
-   * @param listOnlyInState boolean to indicate that the instances should
-   * only include those in a YARN state
-   *  minAppState = currentState = maxAppState 
-   *
-   * @param minAppState minimum application state to include in enumeration.
-   * @param maxAppState maximum application state to include
-   * @return a map of application instance name to description
-   * @throws IOException Any IO problem
-   * @throws YarnException YARN problems
-   */
-  Map enumSliderInstances(
-  boolean listOnlyInState,
-  YarnApplicationState minAppState,
-  YarnApplicationState maxAppState)
-  throws IOException, YarnException;
 
-  /**
-   * Implement the islive action: probe for a cluster of the given name 
existing
-   * @return exit code
-   */
-  int actionFlex(String name, ActionFlexArgs args) throws YarnException, 
IOException;
+  void actionFlex(String name, ActionFlexArgs args) throws YarnException, 
IOException;
 
   /**
* Test for a cluster existing probe for a cluster of the given name existing
@@ -288,7 +231,7 @@ public interface SliderClientAPI extends Service {
* @throws YarnException
* 

[29/50] [abbrv] hadoop git commit: Updated pom to point to 3.0.0-alpha3-SNAPSHOT

[jianhe]

Updated pom to point to 3.0.0-alpha3-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef10c964
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef10c964
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef10c964

Branch: refs/heads/yarn-native-services
Commit: ef10c964932813667edd5b2956068a02f0b11bb7
Parents: 2b808f2
Author: Jian He 
Authored: Tue Feb 7 14:38:32 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml| 4 ++--
 .../hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml   | 2 +-
 .../hadoop-yarn-applications/hadoop-yarn-slider/pom.xml  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef10c964/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index e467f7f..4e88aef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -19,12 +19,12 @@
   
 org.apache.hadoop
 hadoop-yarn-applications
-3.0.0-alpha2-SNAPSHOT
+3.0.0-alpha3-SNAPSHOT
   
   org.apache.hadoop
   hadoop-yarn-services-api
   Apache Hadoop YARN Services API
-  3.0.0-alpha2-SNAPSHOT
+  3.0.0-alpha3-SNAPSHOT
   jar
   Hadoop YARN REST APIs for services
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef10c964/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 81e607b..084a10b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -19,7 +19,7 @@
   
 org.apache.hadoop
 hadoop-yarn-slider
-3.0.0-alpha2-SNAPSHOT
+3.0.0-alpha3-SNAPSHOT
   
   org.apache.hadoop
   hadoop-yarn-slider-core

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef10c964/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
index 780f68b..cc42b18 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
@@ -19,7 +19,7 @@
 
 hadoop-yarn-applications
 org.apache.hadoop
-3.0.0-alpha2-SNAPSHOT
+3.0.0-alpha3-SNAPSHOT
 
 4.0.0
 org.apache.hadoop


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[42/50] [abbrv] hadoop git commit: YARN-6132. SliderClient bondToCluster should call findInstance with live states filter. Contributed by Gour Saha

[jianhe]

YARN-6132. SliderClient bondToCluster should call findInstance with live states 
filter. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b808f2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b808f2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b808f2c

Branch: refs/heads/yarn-native-services
Commit: 2b808f2c452569fcacf8d79c66fce04daf723396
Parents: 5f28a2a
Author: Billie Rinaldi 
Authored: Mon Jan 30 09:20:57 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../src/main/java/org/apache/slider/client/SliderClient.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b808f2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 032eb6e..e84809a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -3519,7 +3519,8 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 if (clustername == null) {
   throw unknownClusterException("(undefined)");
 }
-ApplicationReport instance = findInstance(clustername);
+ApplicationReport instance = findInstance(clustername,
+SliderUtils.getAllLiveAppStates());
 if (null == instance) {
   throw unknownClusterException(clustername);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[02/50] [abbrv] hadoop git commit: YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha

[jianhe]

YARN-5883 Avoid or eliminate expensive YARN get all applications call. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b544bff2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b544bff2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b544bff2

Branch: refs/heads/yarn-native-services
Commit: b544bff2b10a9ab053c8e90137b49f37244a648f
Parents: 7d6162d
Author: Billie Rinaldi 
Authored: Tue Nov 29 13:14:45 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../org/apache/slider/client/SliderClient.java  |  6 +-
 .../slider/client/SliderYarnClientImpl.java | 94 +---
 .../apache/slider/common/tools/SliderUtils.java | 37 +++-
 .../slider/core/registry/YarnAppListClient.java | 21 -
 .../providers/AbstractClientProvider.java   | 28 +-
 .../providers/docker/DockerClientProvider.java  |  8 ++
 .../servicemonitor/YarnApplicationProbe.java| 12 +--
 7 files changed, 160 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b544bff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ea10ed0..3f2df0a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -198,6 +198,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -2083,7 +2084,7 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 
 // add the tags if available
 Set applicationTags = provider.getApplicationTags(sliderFileSystem,
-appOperations);
+appOperations, clustername);
 
 Credentials credentials = null;
 if (clusterSecure) {
@@ -3031,9 +3032,10 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
 } else {
   // scan for instance in single --state state
-  List userInstances = 
yarnClient.listDeployedInstances("");
   state = state.toUpperCase(Locale.ENGLISH);
   YarnApplicationState desiredState = extractYarnApplicationState(state);
+  List userInstances = yarnClient
+  .listDeployedInstances("", EnumSet.of(desiredState), name);
   ApplicationReport foundInstance =
   yarnClient.findAppInInstanceList(userInstances, name, desiredState);
   if (foundInstance != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b544bff2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index d471cdb..258ef31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -53,6 +53,8 @@ import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
   public List listDeployedInstances(String user)
 throws YarnException, 

[28/50] [abbrv] hadoop git commit: YARN-6192. Move yarn-native-service API records into hadoop-yarn-slider-core module. Contributed by Jian He

[jianhe]

YARN-6192. Move yarn-native-service API records into hadoop-yarn-slider-core 
module. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23d557d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23d557d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23d557d2

Branch: refs/heads/yarn-native-services
Commit: 23d557d2ce39f43b64c44fb24aa13435ea50adb5
Parents: 7c90af1
Author: Billie Rinaldi 
Authored: Tue Feb 28 14:45:20 2017 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:19 2017 +0800

--
 .../dev-support/findbugs-exclude.xml|   4 +-
 .../yarn/services/api/ApplicationApi.java   |   2 +-
 .../api/impl/ApplicationApiService.java |  20 +-
 .../yarn/services/resource/Application.java | 453 ---
 .../services/resource/ApplicationState.java |  30 --
 .../services/resource/ApplicationStatus.java| 145 --
 .../hadoop/yarn/services/resource/Artifact.java | 157 ---
 .../yarn/services/resource/BaseResource.java|  48 --
 .../yarn/services/resource/Component.java   | 381 
 .../yarn/services/resource/ConfigFile.java  | 192 
 .../yarn/services/resource/Configuration.java   | 149 --
 .../yarn/services/resource/Container.java   | 294 
 .../yarn/services/resource/ContainerState.java  |  25 -
 .../hadoop/yarn/services/resource/Error.java| 125 -
 .../yarn/services/resource/PlacementPolicy.java |  99 
 .../yarn/services/resource/ReadinessCheck.java  | 163 ---
 .../hadoop/yarn/services/resource/Resource.java | 149 --
 .../src/main/webapp/WEB-INF/web.xml |   2 +-
 .../api/impl/TestApplicationApiService.java |   6 +-
 .../hadoop-yarn-slider-core/pom.xml |   5 +
 .../apache/slider/api/resource/Application.java | 449 ++
 .../slider/api/resource/ApplicationState.java   |  30 ++
 .../slider/api/resource/ApplicationStatus.java  | 145 ++
 .../apache/slider/api/resource/Artifact.java| 157 +++
 .../slider/api/resource/BaseResource.java   |  48 ++
 .../apache/slider/api/resource/Component.java   | 381 
 .../apache/slider/api/resource/ConfigFile.java  | 192 
 .../slider/api/resource/Configuration.java  | 149 ++
 .../apache/slider/api/resource/Container.java   | 294 
 .../slider/api/resource/ContainerState.java |  25 +
 .../org/apache/slider/api/resource/Error.java   | 125 +
 .../slider/api/resource/PlacementPolicy.java|  99 
 .../slider/api/resource/ReadinessCheck.java | 163 +++
 .../apache/slider/api/resource/Resource.java| 149 ++
 34 files changed, 2427 insertions(+), 2428 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d557d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
index 2843338..b89146a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -16,7 +16,5 @@
limitations under the License.
 -->
 
-
-
-
+
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/23d557d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
index 654413c..0fb6402 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/ApplicationApi.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.services.api;
 
 import javax.ws.rs.core.Response;
 
-import org.apache.hadoop.yarn.services.resource.Application;
+import org.apache.slider.api.resource.Application;
 
 /**
  * 

[22/50] [abbrv] hadoop git commit: YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He

[jianhe]

YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e82a134
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e82a134
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e82a134

Branch: refs/heads/yarn-native-services
Commit: 2e82a13494b745349ff23cddb9ad58e87ef09b3b
Parents: 602b154
Author: Billie Rinaldi 
Authored: Mon Dec 19 12:09:53 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../dev-support/findbugs-exclude.xml|  96 +
 .../dev-support/findbugs-exclude.xml|  20 -
 .../hadoop-yarn-slider-core/pom.xml |   5 +
 .../apache/hadoop/security/KerberosDiags.java   |   5 +-
 .../slider/api/proto/RestTypeMarshalling.java   | 283 -
 .../slider/api/types/ComponentInformation.java  |   2 +-
 .../slider/api/types/RestTypeMarshalling.java   | 284 +
 .../org/apache/slider/client/SliderClient.java  |  56 +--
 .../apache/slider/client/TokensOperation.java   |   9 +-
 .../client/ipc/SliderClusterOperations.java |   2 +-
 .../rest/SliderApplicationApiRestClient.java|  14 -
 .../org/apache/slider/common/SliderKeys.java|   8 +-
 .../common/params/AbstractActionArgs.java   |   5 +-
 .../AbstractClusterBuildingActionArgs.java  |   1 -
 .../apache/slider/common/params/CommonArgs.java |   9 +-
 .../apache/slider/common/tools/Comparators.java |  17 +-
 .../slider/common/tools/ConfigHelper.java   |   2 +-
 .../slider/common/tools/CoreFileSystem.java |  31 --
 .../apache/slider/common/tools/SliderUtils.java | 102 +
 .../slider/core/buildutils/InstanceBuilder.java |   1 -
 .../apache/slider/core/conf/AggregateConf.java  |   2 +-
 .../org/apache/slider/core/conf/ConfTree.java   |  15 +-
 .../core/persist/AppDefinitionPersister.java|  15 +-
 .../core/registry/retrieve/AMWebClient.java |  51 ---
 .../core/restclient/HttpOperationResponse.java  |  34 --
 .../restclient/UrlConnectionOperations.java | 120 --
 .../slider/core/zk/MiniZooKeeperCluster.java|  27 +-
 .../apache/slider/core/zk/ZKIntegration.java|  20 +-
 .../apache/slider/providers/ProviderUtils.java  |  21 +-
 .../providers/docker/DockerProviderService.java |  12 +-
 .../server/appmaster/management/BoolMetric.java |  15 -
 .../management/MetricsAndMonitoring.java|  25 --
 .../management/RangeLimitedCounter.java |  85 
 .../appmaster/management/RecordedEvent.java |  58 ---
 .../server/appmaster/rpc/SliderIPCService.java  |   2 +-
 .../server/appmaster/state/NodeInstance.java|   2 +-
 .../state/OutstandingRequestTracker.java|   2 +-
 .../server/appmaster/state/RoleStatus.java  |   4 -
 .../appmaster/web/rest/InsecureAmFilter.java|   1 -
 .../web/rest/publisher/PublisherResource.java   |   5 +-
 .../appmaster/web/view/ContainerStatsBlock.java |   3 +-
 .../server/servicemonitor/ProbeStatus.java  |   1 +
 .../utility/AbstractSliderLaunchedService.java  |   2 -
 .../services/workflow/LongLivedProcess.java |   5 +-
 .../slider/common/tools/TestSliderUtils.java|   2 +-
 .../slider/test/MiniZooKeeperCluster.java   | 395 ---
 46 files changed, 501 insertions(+), 1375 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e82a134/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
new file mode 100644
index 000..d253762
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -0,0 +1,96 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

[38/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
index 7bfb410..abad34e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/services/api/impl/TestApplicationApiService.java
@@ -17,16 +17,15 @@
 
 package org.apache.hadoop.yarn.services.api.impl;
 
-import static org.apache.hadoop.yarn.services.utils.RestApiConstants.*;
-import static org.apache.hadoop.yarn.services.utils.RestApiErrorMessages.*;
+import static org.apache.slider.util.RestApiConstants.*;
+import static org.apache.slider.util.RestApiErrorMessages.*;
 
-import java.util.HashMap;
-import java.util.Map;
+import java.util.ArrayList;
 
 import org.apache.slider.api.resource.Application;
 import org.apache.slider.api.resource.Artifact;
 import org.apache.slider.api.resource.Resource;
-import org.apache.slider.common.SliderKeys;
+import org.apache.slider.util.ServiceApiUtil;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -61,12 +60,10 @@ public class TestApplicationApiService {
   @Test(timeout = 9)
   public void testValidateApplicationPostPayload() throws Exception {
 Application app = new Application();
-Map compNameArtifactIdMap = new HashMap<>();
 
 // no name
 try {
-  appApiService.validateApplicationPostPayload(app,
-  compNameArtifactIdMap);
+  ServiceApiUtil.validateApplicationPostPayload(app);
   Assert.fail(EXCEPTION_PREFIX + "application with no name");
 } catch (IllegalArgumentException e) {
   Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID, e.getMessage());
@@ -77,8 +74,7 @@ public class TestApplicationApiService {
 for (String badName : badNames) {
   app.setName(badName);
   try {
-appApiService.validateApplicationPostPayload(app,
-compNameArtifactIdMap);
+ServiceApiUtil.validateApplicationPostPayload(app);
 Assert.fail(EXCEPTION_PREFIX + "application with bad name " + badName);
   } catch (IllegalArgumentException e) {
 Assert.assertEquals(ERROR_APPLICATION_NAME_INVALID_FORMAT,
@@ -89,8 +85,7 @@ public class TestApplicationApiService {
 // no artifact
 app.setName("finance_home");
 try {
-  appApiService.validateApplicationPostPayload(app,
-  compNameArtifactIdMap);
+  ServiceApiUtil.validateApplicationPostPayload(app);
   Assert.fail(EXCEPTION_PREFIX + "application with no artifact");
 } catch (IllegalArgumentException e) {
   Assert.assertEquals(ERROR_ARTIFACT_INVALID, e.getMessage());
@@ -100,8 +95,7 @@ public class TestApplicationApiService {
 Artifact artifact = new Artifact();
 app.setArtifact(artifact);
 try {
-  appApiService.validateApplicationPostPayload(app,
-  compNameArtifactIdMap);
+  ServiceApiUtil.validateApplicationPostPayload(app);
   Assert.fail(EXCEPTION_PREFIX + "application with no artifact id");
 } catch (IllegalArgumentException e) {
   Assert.assertEquals(ERROR_ARTIFACT_ID_INVALID, e.getMessage());
@@ -112,8 +106,7 @@ public class TestApplicationApiService {
 artifact.setId("app.io/hbase:facebook_0.2");
 app.setNumberOfContainers(5l);
 try {
-  appApiService.validateApplicationPostPayload(app,
-  compNameArtifactIdMap);
+  ServiceApiUtil.validateApplicationPostPayload(app);
 } catch (IllegalArgumentException e) {
   logger.error("application attributes specified should be valid here", e);
   Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
@@ -124,22 +117,18 @@ public class TestApplicationApiService {
 Assert.assertEquals(app.getComponents().get(0).getName(),
 DEFAULT_COMPONENT_NAME);
 Assert.assertEquals(app.getLifetime(), DEFAULT_UNLIMITED_LIFETIME);
-Assert.assertEquals("Property not set",
-app.getConfiguration().getProperties()
-.get(SliderKeys.COMPONENT_TYPE_KEY),
-SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
+//TODO handle external app
 
 // unset artifact type, default component and no of containers to test 
other
 // validation logic
 artifact.setType(null);
-app.setComponents(null);
+app.setComponents(new 

[19/50] [abbrv] hadoop git commit: YARN-6014. Followup fix for slider core module findbugs. Contributed by Jian He

[jianhe]

YARN-6014. Followup fix for slider core module findbugs. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/18fb4a59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/18fb4a59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/18fb4a59

Branch: refs/heads/yarn-native-services
Commit: 18fb4a59d9f37e168947258d2535b2954cd6cb94
Parents: 9c4d2f8
Author: Billie Rinaldi 
Authored: Tue Dec 20 10:59:58 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../dev-support/findbugs-exclude.xml   | 17 +
 .../web/rest/application/actions/StopResponse.java |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/18fb4a59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
index d253762..e5cde4e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -71,6 +71,23 @@
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/18fb4a59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
index 9af6a22..d591f57 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/actions/StopResponse.java
@@ -24,6 +24,6 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
 @JsonIgnoreProperties(ignoreUnknown = true)
 @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
 public class StopResponse {
-  String verb;
+  public String verb;
   public String text;
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[49/50] [abbrv] hadoop git commit: YARN-6395. Integrate service app master to write data into ATSv2. Contributed by Rohith Sharma K S

[jianhe]

YARN-6395. Integrate service app master to write data into ATSv2. Contributed 
by Rohith Sharma K S


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/538658d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/538658d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/538658d2

Branch: refs/heads/yarn-native-services
Commit: 538658d2281bdac19652d0aa1e3d6a55136cfff6
Parents: 821abcd
Author: Jian He 
Authored: Thu Mar 30 15:58:51 2017 +0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:20 2017 +0800

--
 .../server/appmaster/SliderAppMaster.java   |  63 
 .../slider/server/appmaster/state/AppState.java |  11 +
 .../ServiceTimelinePublisher.java   | 365 +++
 .../timelineservice/SliderMetricsSink.java  | 102 ++
 .../SliderTimelineEntityType.java   |  39 ++
 .../timelineservice/SliderTimelineEvent.java|  34 ++
 .../SliderTimelineMetricsConstants.java |  91 +
 .../appmaster/timelineservice/package-info.java |  27 ++
 .../TestServiceTimelinePublisher.java   | 285 +++
 .../appmaster/timelineservice/package-info.java |  26 ++
 10 files changed, 1043 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/538658d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 4922c2d..4fa2769 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
@@ -146,6 +147,8 @@ import 
org.apache.slider.server.appmaster.state.ContainerAssignment;
 import 
org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector;
 import org.apache.slider.server.appmaster.state.ProviderAppState;
 import org.apache.slider.server.appmaster.state.RoleInstance;
+import 
org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
+import org.apache.slider.server.appmaster.timelineservice.SliderMetricsSink;
 import org.apache.slider.server.appmaster.web.SliderAMWebApp;
 import org.apache.slider.server.appmaster.web.WebAppApi;
 import org.apache.slider.server.appmaster.web.WebAppApiImpl;
@@ -240,6 +243,13 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
   @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
   private AMRMClientAsync asyncRMClient;
 
+  /** Handle to communicate with the timeline service */
+  private TimelineClient timelineClient;
+
+  private boolean timelineServiceEnabled = false;
+
+  ServiceTimelinePublisher serviceTimelinePublisher;
+
   @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
   private RMOperationHandler rmOperationHandler;
 
@@ -483,6 +493,10 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 addService(executorService);
 
 addService(actionQueues);
+if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+  timelineServiceEnabled = true;
+  log.info("Enabled YARN timeline service v2. ");
+}
 
 //init all child services
 super.serviceInit(conf);
@@ -650,6 +664,20 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
   //now bring it up
   deployChildService(asyncRMClient);
 
+  if (timelineServiceEnabled) {
+timelineClient = TimelineClient.createTimelineClient(appid);
+asyncRMClient.registerTimelineClient(timelineClient);
+timelineClient.init(getConfig());
+timelineClient.start();
+log.info("Timeline client started.");
+
+serviceTimelinePublisher = 

[37/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 00e2b62..f4ea70b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -20,7 +20,6 @@ package org.apache.slider.client;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.Files;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
@@ -35,7 +34,6 @@ import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.registry.client.api.RegistryConstants;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
@@ -55,40 +53,44 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
 import 
org.apache.hadoop.yarn.api.protocolrecords.UpdateApplicationTimeoutsRequest;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
 import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
 import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
 import org.apache.hadoop.yarn.util.Times;
-import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.ClusterNode;
 import org.apache.slider.api.SliderApplicationApi;
 import org.apache.slider.api.SliderClusterProtocol;
-import org.apache.slider.api.StateValues;
 import org.apache.slider.api.proto.Messages;
+import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.Component;
 import org.apache.slider.api.types.ContainerInformation;
 import org.apache.slider.api.types.NodeInformationList;
-import org.apache.slider.api.types.SliderInstanceDescription;
 import org.apache.slider.client.ipc.SliderApplicationIpcClient;
 import org.apache.slider.client.ipc.SliderClusterOperations;
 import org.apache.slider.common.Constants;
 import org.apache.slider.common.SliderExitCodes;
 import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.SliderXmlConfKeys;
 import org.apache.slider.common.params.AbstractActionArgs;
 import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
 import org.apache.slider.common.params.ActionAMSuicideArgs;
 import org.apache.slider.common.params.ActionClientArgs;
 import org.apache.slider.common.params.ActionCreateArgs;
 import org.apache.slider.common.params.ActionDependencyArgs;
-import org.apache.slider.common.params.ActionDestroyArgs;
 import org.apache.slider.common.params.ActionDiagnosticArgs;
 import org.apache.slider.common.params.ActionEchoArgs;
 import org.apache.slider.common.params.ActionExistsArgs;
@@ -113,20 +115,13 @@ import org.apache.slider.common.params.ActionUpgradeArgs;
 import org.apache.slider.common.params.Arguments;
 import org.apache.slider.common.params.ClientArgs;
 import org.apache.slider.common.params.CommonArgs;
-import org.apache.slider.common.params.LaunchArgsAccessor;
 import 

[16/50] [abbrv] hadoop git commit: YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5712253c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
deleted file mode 100644
index e82ad84..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-//import java.nio.file.Files;
-//import java.nio.file.Path;
-//import java.nio.file.Paths;
-//import java.nio.file.attribute.PosixFilePermission;
-//import java.nio.file.attribute.PosixFilePermissions;
-
-
-/**
- *
- */
-public class SecurityUtils {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(SecurityUtils.class);
-
-  private static String CA_CONFIG_CONTENTS =  "HOME= .\n"
-+ "RANDFILE= 
$ENV::HOME/.rnd\n\n"
-+ "[ ca ]\n"
-+ "default_ca = 
CA_CLIENT\n"
-+ "[ CA_CLIENT ]\n"
-+ "dir= 
${SEC_DIR}/db\n"
-+ "certs  = 
$dir/certs\n"
-+ "new_certs_dir  = 
$dir/newcerts\n"
-+ "\n"
-+ "database   = 
$dir/index.txt\n"
-+ "serial = 
$dir/serial\n"
-+ "default_days   = 365
\n"
-+ "\n"
-+ "default_crl_days   = 7  \n"
-+ "default_md = sha256 
\n"
-+ "\n"
-+ "policy = 
policy_anything \n"
-+ "\n"
-+ "[ policy_anything ]\n"
-+ "countryName= 
optional\n"
-+ "stateOrProvinceName= 
optional\n"
-+ "localityName   = 
optional\n"
-+ "organizationName   = 
optional\n"
-+ "organizationalUnitName = 
optional\n"
-+ "commonName = 
optional\n"
-+ "emailAddress   = 
optional\n"
-+ "\n"
-+ "[req]\n"
-+ "distinguished_name = 
req_distinguished_name\n"
-+ 

[34/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
index 86d87ac..8b88c28 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
@@ -22,7 +22,6 @@ import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.core.conf.AggregateConf;
 import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.providers.AbstractClientProvider;
 import org.apache.slider.providers.ProviderRole;
@@ -30,14 +29,10 @@ import org.apache.slider.providers.ProviderUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
-import static org.apache.slider.providers.docker.DockerKeys.DOCKER_IMAGE;
-
 public class DockerClientProvider extends AbstractClientProvider
 implements SliderKeys {
 
@@ -64,35 +59,7 @@ public class DockerClientProvider extends 
AbstractClientProvider
   public void validateInstanceDefinition(AggregateConf instanceDefinition,
   SliderFileSystem fs) throws SliderException {
 super.validateInstanceDefinition(instanceDefinition, fs);
-
-ConfTreeOperations appConf = instanceDefinition.getAppConfOperations();
-ConfTreeOperations resources = instanceDefinition.getResourceOperations();
-
-for (String roleGroup : resources.getComponentNames()) {
-  if (roleGroup.equals(COMPONENT_AM)) {
-continue;
-  }
-  if (appConf.getComponentOpt(roleGroup, DOCKER_IMAGE, null) == null &&
-  appConf.getGlobalOptions().get(DOCKER_IMAGE) == null) {
-throw new BadConfigException("Property " + DOCKER_IMAGE + " not " +
-"specified for " + roleGroup);
-  }
-
-  providerUtils.getPackages(roleGroup, appConf);
-
-  if (appConf.getComponentOptBool(roleGroup, AM_CONFIG_GENERATION, false)) 
{
-// build and localize configuration files
-Map> configurations =
-providerUtils.buildConfigurations(appConf, appConf, null,
-null, roleGroup, roleGroup, null);
-try {
-  providerUtils.localizeConfigFiles(null, roleGroup, roleGroup, 
appConf,
-  configurations, null, fs, null);
-} catch (IOException e) {
-  throw new BadConfigException(e.toString());
-}
-  }
-}
+//TODO validate Application payload, part of that is already done in 
ApplicationApiService, need to do more
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index 63416cc..511f7bc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -17,286 +17,129 @@
  */
 package org.apache.slider.providers.docker;
 
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import 

[20/50] [abbrv] hadoop git commit: YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He

[jianhe]

YARN-5740. Add a new field in Slider status output - lifetime (remaining). 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea0d63ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea0d63ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea0d63ec

Branch: refs/heads/yarn-native-services
Commit: ea0d63ec8629f3daac5b50f8212f998b5232e745
Parents: 5712253
Author: Gour Saha 
Authored: Fri Dec 16 12:03:51 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../api/impl/ApplicationApiService.java | 82 
 .../org/apache/slider/client/SliderClient.java  | 62 +++
 .../slider/common/params/ActionStatusArgs.java  |  4 +
 3 files changed, 103 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea0d63ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index c4f5d43..b11da2c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -52,6 +52,8 @@ import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.services.api.ApplicationApi;
@@ -771,7 +773,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 
 // Get all applications in a specific state - lighter projection. For full
 // detail, call getApplication on a specific app.
-Set applications;
+Set applications;
 try {
   if (StringUtils.isNotEmpty(state)) {
 ApplicationStatus appStatus = new ApplicationStatus();
@@ -793,13 +795,12 @@ public class ApplicationApiService implements 
ApplicationApi {
 Set apps = new HashSet();
 if (applications.size() > 0) {
   try {
-for (String app : applications) {
+for (ApplicationReport app : applications) {
   Application application = new Application();
-  // TODO: Need to get lifetime, launch-time and privileged container
-  // status from YARN
-  application.setLifetime(null);
-  application.setLaunchTime(new Date());
-  application.setName(app);
+  application.setLifetime(app.getApplicationTimeouts().get(
+  ApplicationTimeoutType.LIFETIME).getRemainingTime());
+  application.setLaunchTime(new Date(app.getStartTime()));
+  application.setName(app.getName());
   // Containers not required, setting to null to avoid empty list
   application.setContainers(null);
   apps.add(application);
@@ -930,9 +931,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 app.setLaunchTime(appStatus.get("createTime") == null ? null
 : new Date(appStatus.get("createTime").getAsLong()));
 
-// lifetime - set it to unlimited for now
-// TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN
-app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+app.setLifetime(queryLifetime(appName));
 
 // Quicklinks
 Map appQuicklinks = new HashMap<>();
@@ -1062,6 +1061,24 @@ public class ApplicationApiService implements 
ApplicationApi {
 return object.get(key) == null ? null : object.get(key).getAsJsonObject();
   }
 
+  private long queryLifetime(String appName) {
+try {
+  return invokeSliderClientRunnable(
+  new SliderClientContextRunnable() {
+@Override
+public Long run(SliderClient sliderClient)
+throws YarnException, IOException, InterruptedException {
+  ApplicationReport report = sliderClient.findInstance(appName);
+  

[15/50] [abbrv] hadoop git commit: YARN-5993. Allow native services quicklinks to be exported for each component. Contributed by Billie Rinaldi

[jianhe]

YARN-5993. Allow native services quicklinks to be exported for each component. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4faaf9f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4faaf9f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4faaf9f3

Branch: refs/heads/yarn-native-services
Commit: 4faaf9f326a4bd56db7f5a40f5b550180fe5f047
Parents: c64e27a
Author: Gour Saha 
Authored: Wed Dec 28 15:24:36 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../apache/slider/common/tools/SliderUtils.java |   8 +-
 .../core/registry/docstore/ExportEntry.java |  20 +++
 .../registry/docstore/PublishedExports.java |  23 ++-
 .../apache/slider/providers/ProviderUtils.java  | 133 
 .../providers/docker/DockerProviderService.java | 151 ---
 .../server/appmaster/web/view/IndexBlock.java   |  15 +-
 6 files changed, 177 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index a107dfb..20c7831 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -183,17 +183,17 @@ public final class SliderUtils {
 return !isUnset(s);
   }
 
-  public static boolean isEmpty(List l) {
+  public static boolean isEmpty(Collection l) {
 return l == null || l.isEmpty();
   }
 
   /**
-   * Probe for a list existing and not being empty
-   * @param l list
+   * Probe for a collection existing and not being empty
+   * @param l collection
* @return true if the reference is valid and it contains entries
*/
 
-  public static boolean isNotEmpty(List l) {
+  public static boolean isNotEmpty(Collection l) {
 return l != null && !l.isEmpty();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
index 4bcf6c1..dd6e034 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -107,6 +107,26 @@ public class ExportEntry {
   }
 
   @Override
+  public boolean equals(Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+
+ExportEntry that = (ExportEntry) o;
+
+if (value != null ? !value.equals(that.value) : that.value != null)
+  return false;
+return containerId != null ? containerId.equals(that.containerId) :
+that.containerId == null;
+  }
+
+  @Override
+  public int hashCode() {
+int result = value != null ? value.hashCode() : 0;
+result = 31 * result + (containerId != null ? containerId.hashCode() : 0);
+return result;
+  }
+
+  @Override
   public String toString() {
 return new StringBuilder("ExportEntry{").
 append("value='").append(value).append("',").

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4faaf9f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java

[25/50] [abbrv] hadoop git commit: YARN-6010. Fix findbugs, site warnings in yarn-services-api module. Contributed by Jian He

[jianhe]

YARN-6010. Fix findbugs, site warnings in yarn-services-api module. Contributed 
by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c4d2f87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c4d2f87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c4d2f87

Branch: refs/heads/yarn-native-services
Commit: 9c4d2f87835edfd1fa8d407724838670fb3f00a4
Parents: 6053c69
Author: Billie Rinaldi 
Authored: Mon Dec 19 16:15:58 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../dev-support/findbugs-exclude.xml   |  4 +++-
 .../hadoop-yarn-services-api/pom.xml   | 13 -
 2 files changed, 3 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4d2f87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
index b89146a..2843338 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -16,5 +16,7 @@
limitations under the License.
 -->
 
-
+
+
+
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4d2f87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index c198f28..e467f7f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -108,19 +108,6 @@
 
   
 
-  
-org.apache.maven.plugins
-maven-site-plugin
-${maven-site-plugin.version}
-
-
-  
-org.apache.maven.doxia
-doxia-module-markdown
-${maven-doxia-module-markdown.version}
-  
-
-  
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[21/50] [abbrv] hadoop git commit: YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e82a134/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
index c8b3adb..1af883e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
@@ -137,8 +137,6 @@ public class MiniZooKeeperCluster extends AbstractService {
   }
 
   /**
-   * @param baseDir
-   * @param numZooKeeperServers
* @return ClientPort server bound to, -1 if there was a
* binding problem and we couldn't pick another port.
* @throws IOException
@@ -229,17 +227,6 @@ public class MiniZooKeeperCluster extends AbstractService {
 }
   }
 
-  /**
-   * Delete the basedir
-   */
-  private void deleteBaseDir() {
-if (baseDir != null) {
-  baseDir.delete();
-  baseDir = null;
-}
-
-  }
-
   @Override
   protected void serviceStop() throws Exception {
 
@@ -359,7 +346,7 @@ public class MiniZooKeeperCluster extends AbstractService {
 try {
   sock = new Socket("localhost", port);
   OutputStream outstream = sock.getOutputStream();
-  outstream.write("stat".getBytes());
+  outstream.write("stat".getBytes("UTF-8"));
   outstream.flush();
 } finally {
   IOUtils.closeSocket(sock);
@@ -387,10 +374,10 @@ public class MiniZooKeeperCluster extends AbstractService 
{
 BufferedReader reader = null;
 try {
   OutputStream outstream = sock.getOutputStream();
-  outstream.write("stat".getBytes());
+  outstream.write("stat".getBytes("UTF-8"));
   outstream.flush();
 
-  Reader isr = new InputStreamReader(sock.getInputStream());
+  Reader isr = new InputStreamReader(sock.getInputStream(), "UTF-8");
   reader = new BufferedReader(isr);
   String line = reader.readLine();
   if (line != null && line.startsWith("Zookeeper version:")) {
@@ -412,12 +399,4 @@ public class MiniZooKeeperCluster extends AbstractService {
 }
 return false;
   }
-
-  public int getClientPort() {
-return clientPort;
-  }
-
-  public String getZkQuorum() {
-return zkQuorum;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e82a134/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 6ed58d5..ef96c9b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -43,14 +43,14 @@ public class ZKIntegration implements Watcher, Closeable {
 /**
  * Base path for services
  */
-  public static String ZK_SERVICES = "services";
+  public static final String ZK_SERVICES = "services";
   /**
* Base path for all Slider references
*/
-  public static String ZK_SLIDER = "slider";
-  public static String ZK_USERS = "users";
-  public static String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
-  public static String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+  public static final String ZK_SLIDER = "slider";
+  public static final String ZK_USERS = "users";
+  public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+  public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
 
   public static final List ZK_USERS_PATH_LIST = new 
ArrayList();
   static {
@@ -59,7 +59,7 @@ public class ZKIntegration implements Watcher, Closeable {
 ZK_USERS_PATH_LIST.add(ZK_USERS);
   }
 
-  public static int SESSION_TIMEOUT = 3;
+  public static final int SESSION_TIMEOUT = 3;
   

[33/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index f67ea58..4922c2d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
 import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import org.apache.hadoop.http.HttpConfig;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.registry.client.api.RegistryOperations;
 import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
 import org.apache.hadoop.registry.client.binding.RegistryTypeUtils;
@@ -62,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
@@ -77,13 +80,12 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.WebAppException;
 import org.apache.hadoop.yarn.webapp.WebApps;
 import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.apache.slider.api.ClusterDescription;
 import org.apache.slider.api.InternalKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.StatusKeys;
+import org.apache.slider.api.proto.Messages;
 import org.apache.slider.api.proto.SliderClusterAPI;
-import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.api.resource.Application;
 import org.apache.slider.common.SliderExitCodes;
 import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.params.AbstractActionArgs;
@@ -95,10 +97,7 @@ import org.apache.slider.common.tools.PortScanner;
 import org.apache.slider.common.tools.SliderFileSystem;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.common.tools.SliderVersionInfo;
-import org.apache.slider.core.buildutils.InstanceIO;
 import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.conf.ConfTreeOperations;
 import org.apache.slider.core.conf.MapOperations;
 import org.apache.slider.core.exceptions.BadConfigException;
 import org.apache.slider.core.exceptions.SliderException;
@@ -109,13 +108,12 @@ import org.apache.slider.core.main.ExitCodeProvider;
 import org.apache.slider.core.main.LauncherExitCodes;
 import org.apache.slider.core.main.RunService;
 import org.apache.slider.core.main.ServiceLauncher;
+import org.apache.slider.core.persist.JsonSerDeser;
 import org.apache.slider.core.registry.info.CustomRegistryConstants;
 import org.apache.slider.providers.ProviderCompleted;
 import org.apache.slider.providers.ProviderRole;
 import org.apache.slider.providers.ProviderService;
 import org.apache.slider.providers.SliderProviderFactory;
-import org.apache.slider.providers.slideram.SliderAMClientProvider;
-import org.apache.slider.providers.slideram.SliderAMProviderService;
 import org.apache.slider.server.appmaster.actions.ActionHalt;
 import 
org.apache.slider.server.appmaster.actions.ActionRegisterServiceInstance;
 import org.apache.slider.server.appmaster.actions.ActionStopSlider;
@@ -136,7 +134,6 @@ import 
org.apache.slider.server.appmaster.monkey.ChaosKillContainer;
 import org.apache.slider.server.appmaster.monkey.ChaosMonkeyService;
 import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
 import org.apache.slider.server.appmaster.operations.AsyncRMOperationHandler;
-import 
org.apache.slider.server.appmaster.operations.ProviderNotifyingOperationHandler;
 import org.apache.slider.server.appmaster.operations.RMOperationHandler;
 import org.apache.slider.server.appmaster.rpc.RpcBinder;
 import 

[23/50] [abbrv] hadoop git commit: YARN-5968. Fix slider core module javadocs. Contributed by Billie Rinaldi

[jianhe]

YARN-5968. Fix slider core module javadocs. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6053c69c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6053c69c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6053c69c

Branch: refs/heads/yarn-native-services
Commit: 6053c69c0591d49c6c13a0c9dc3ea524575c83a6
Parents: 2e82a13
Author: Jian He 
Authored: Mon Dec 19 13:06:56 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6053c69c/hadoop-yarn-project/hadoop-yarn/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/pom.xml
index c43588a..4504140 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
 org.apache.maven.plugins
 maven-javadoc-plugin
 
-  
org.apache.hadoop.yarn.proto
+  
org.apache.hadoop.yarn.proto,org.apache.slider
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[03/50] [abbrv] hadoop git commit: YARN-5769. Integrate update app lifetime using feature implemented in YARN-5611. Contributed by Jian He

[jianhe]

YARN-5769. Integrate update app lifetime using feature implemented in 
YARN-5611. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62154ae8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62154ae8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62154ae8

Branch: refs/heads/yarn-native-services
Commit: 62154ae8449baaccefb922e68b8c777ba56811ea
Parents: a6ed5f1
Author: Gour Saha 
Authored: Wed Nov 30 23:24:45 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../api/impl/ApplicationApiService.java | 25 +--
 .../org/apache/slider/client/SliderClient.java  | 33 +---
 .../AbstractClusterBuildingActionArgs.java  |  3 +-
 3 files changed, 53 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/62154ae8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 6db69ac..c4f5d43 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -77,6 +77,7 @@ import org.apache.slider.common.params.ActionFreezeArgs;
 import org.apache.slider.common.params.ActionListArgs;
 import org.apache.slider.common.params.ActionRegistryArgs;
 import org.apache.slider.common.params.ActionThawArgs;
+import org.apache.slider.common.params.ActionUpdateArgs;
 import org.apache.slider.common.params.ComponentArgsDelegate;
 import org.apache.slider.common.tools.SliderUtils;
 import org.apache.slider.common.tools.SliderVersionInfo;
@@ -1398,14 +1399,34 @@ public class ApplicationApiService implements 
ApplicationApi {
 }
 
 // If new lifetime value specified then update it
-if (updateAppData.getLifetime() != null) {
-  // TODO: Once YARN-3813 and YARN-4205 are available
+if (updateAppData.getLifetime() != null
+&& updateAppData.getLifetime() > 0) {
+  try {
+updateAppLifetime(appName, updateAppData.getLifetime());
+  } catch (Exception e) {
+logger.error("Failed to update application (" + appName + ") lifetime 
("
++ updateAppData.getLifetime() + ")", e);
+return Response.status(Status.INTERNAL_SERVER_ERROR).build();
+  }
 }
 
 // If nothing happens consider it a no-op
 return Response.status(Status.NO_CONTENT).build();
   }
 
+  private Void updateAppLifetime(String appName, long lifetime)
+  throws InterruptedException, YarnException, IOException {
+return invokeSliderClientRunnable(new SliderClientContextRunnable() {
+  @Override public Void run(SliderClient sliderClient)
+  throws YarnException, IOException, InterruptedException {
+ActionUpdateArgs args = new ActionUpdateArgs();
+args.lifetime = lifetime;
+sliderClient.actionUpdate(appName, args);
+return null;
+  }
+});
+  }
+
   // create default component and initialize with app level global values
   private List getDefaultComponentAsList(Application app) {
 List comps = getDefaultComponentAsList();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/62154ae8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3f2df0a..12f7870 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 

[12/50] [abbrv] hadoop git commit: YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1ce97eb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
new file mode 100644
index 000..52b3c37
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -0,0 +1,1534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.net.util.Base64;
+import org.apache.commons.net.util.SubnetUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.DNSOperations;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.DNSKEYRecord;
+import org.xbill.DNS.DNSSEC;
+import org.xbill.DNS.DSRecord;
+import org.xbill.DNS.ExtendedFlags;
+import org.xbill.DNS.Flags;
+import org.xbill.DNS.Header;
+import org.xbill.DNS.Message;
+import org.xbill.DNS.NSRecord;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.NameTooLongException;
+import org.xbill.DNS.OPTRecord;
+import org.xbill.DNS.Opcode;
+import org.xbill.DNS.RRSIGRecord;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Rcode;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SOARecord;
+import org.xbill.DNS.Section;
+import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.TSIG;
+import org.xbill.DNS.TSIGRecord;
+import org.xbill.DNS.TextParseException;
+import org.xbill.DNS.Type;
+import org.xbill.DNS.Zone;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.RSAPrivateKeySpec;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+
+/**
+ * A DNS service reflecting the state of the YARN registry.  Records are 
created
+ * based on service records available in the YARN ZK-based registry.
+ */
+public class RegistryDNS extends AbstractService implements DNSOperations,
+ZoneSelector {
+
+  public static final String CONTAINER = "container";
+
+  static final int FLAG_DNSSECOK = 1;
+  static final int FLAG_SIGONLY = 2;
+
+  private static final Logger LOG =
+  

[31/50] [abbrv] hadoop git commit: YARN-6255. Refactor yarn-native-services framework. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/821abcd2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 98557ce..8e8546b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -21,19 +21,22 @@ package org.apache.slider.server.appmaster.state;
 import com.codahale.metrics.Metric;
 import com.codahale.metrics.MetricSet;
 import com.google.common.base.Preconditions;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.slider.api.types.ComponentInformation;
 import org.apache.slider.api.types.RoleStatistics;
 import org.apache.slider.providers.PlacementPolicy;
 import org.apache.slider.providers.ProviderRole;
 import org.apache.slider.server.appmaster.management.BoolMetricPredicate;
-import org.apache.slider.server.appmaster.management.LongGauge;
+import org.apache.slider.server.appmaster.metrics.SliderMetrics;
 
 import java.io.Serializable;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
 /**
  * Models the ongoing status of all nodes in an application.
  *
@@ -42,7 +45,7 @@ import java.util.Map;
  * requires synchronization. Where synchronized access is good is that it 
allows for
  * the whole instance to be locked, for updating multiple entries.
  */
-public final class RoleStatus implements Cloneable, MetricSet {
+public final class RoleStatus implements MetricSet {
 
   private final String name;
   private final String group;
@@ -53,25 +56,9 @@ public final class RoleStatus implements Cloneable, 
MetricSet {
   private final int key;
   private final ProviderRole providerRole;
 
-  private final LongGauge actual = new LongGauge();
-  private final LongGauge completed = new LongGauge();
-  private final LongGauge desired = new LongGauge();
-  private final LongGauge failed = new LongGauge();
-  private final LongGauge failedRecently = new LongGauge(0);
-  private final LongGauge limitsExceeded = new LongGauge(0);
-  private final LongGauge nodeFailed = new LongGauge(0);
-  /** Number of AA requests queued. */
-  private final LongGauge pendingAntiAffineRequests = new LongGauge(0);
-  private final LongGauge preempted = new LongGauge(0);
-  private final LongGauge releasing = new LongGauge();
-  private final LongGauge requested = new LongGauge();
-  private final LongGauge started = new LongGauge();
-  private final LongGauge startFailed = new LongGauge();
-  private final LongGauge totalRequested = new LongGauge();
-
   /** resource requirements */
   private Resource resourceRequirements;
-
+  private SliderMetrics componentMetrics;
 
   /** any pending AA request */
   private volatile OutstandingRequest outstandingAArequest = null;
@@ -84,28 +71,19 @@ public final class RoleStatus implements Cloneable, 
MetricSet {
 this.name = providerRole.name;
 this.group = providerRole.group;
 this.key = providerRole.id;
+componentMetrics =
+SliderMetrics.register(this.name, "Metrics for component " + 
this.name);
+componentMetrics
+.tag("type", "Metrics type [component or service]", "component");
+  }
+
+  public SliderMetrics getComponentMetrics() {
+return this.componentMetrics;
   }
 
   @Override
   public Map getMetrics() {
 Map metrics = new HashMap<>(15);
-metrics.put("actual", actual);
-metrics.put("completed", completed );
-metrics.put("desired", desired);
-metrics.put("failed", failed);
-metrics.put("limitsExceeded", limitsExceeded);
-metrics.put("nodeFailed", nodeFailed);
-metrics.put("preempted", preempted);
-metrics.put("pendingAntiAffineRequests", pendingAntiAffineRequests);
-metrics.put("releasing", releasing);
-metrics.put("requested", requested);
-metrics.put("preempted", preempted);
-metrics.put("releasing", releasing );
-metrics.put("requested", requested);
-metrics.put("started", started);
-metrics.put("startFailed", startFailed);
-metrics.put("totalRequested", totalRequested);
-
 metrics.put("outstandingAArequest",
   

[50/50] [abbrv] hadoop git commit: Rebase to latest trunk, fix conflicts and pom.xml

[jianhe]

Rebase to latest trunk, fix conflicts and pom.xml


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ccd2b0b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ccd2b0b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ccd2b0b3

Branch: refs/heads/yarn-native-services
Commit: ccd2b0b353d0384612f44c266a496e382bcf0b62
Parents: db9378f
Author: Jian He 
Authored: Tue Apr 18 14:04:35 2017 +0800
Committer: Jian He 
Committed: Tue Apr 18 14:04:35 2017 +0800

--
 .../hadoop-yarn-slider-core/pom.xml   |  5 +
 .../slider/server/appmaster/SliderAppMaster.java  |  7 ---
 .../timelineservice/ServiceTimelinePublisher.java |  5 +++--
 .../timelineservice/TestServiceTimelinePublisher.java | 14 +++---
 4 files changed, 23 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccd2b0b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index ee77ecb..16a2bb2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -184,6 +184,11 @@
 
 
   org.apache.hadoop
+  hadoop-hdfs-client
+
+
+
+  org.apache.hadoop
   hadoop-yarn-client
   compile
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccd2b0b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index eca07e6..4aeffa6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.client.api.TimelineV2Client;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
 import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
@@ -240,7 +241,7 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
   private AMRMClientAsync asyncRMClient;
 
   /** Handle to communicate with the timeline service */
-  private TimelineClient timelineClient;
+  private TimelineV2Client timelineClient;
 
   private boolean timelineServiceEnabled = false;
 
@@ -658,8 +659,8 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
   deployChildService(asyncRMClient);
 
   if (timelineServiceEnabled) {
-timelineClient = TimelineClient.createTimelineClient(appid);
-asyncRMClient.registerTimelineClient(timelineClient);
+timelineClient = TimelineV2Client.createTimelineClient(appid);
+asyncRMClient.registerTimelineV2Client(timelineClient);
 timelineClient.init(getConfig());
 timelineClient.start();
 log.info("Timeline client started.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ccd2b0b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/timelineservice/ServiceTimelinePublisher.java
 

[14/50] [abbrv] hadoop git commit: YARN-4757. Add the ability to split reverse zone subnets. Contributed by Shane Kumpf.

[jianhe]

YARN-4757. Add the ability to split reverse zone subnets. Contributed by Shane 
Kumpf.

(cherry picked from commit 9bff70f1316d889c3109c68dd308c8f456229bf8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c64e27af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c64e27af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c64e27af

Branch: refs/heads/yarn-native-services
Commit: c64e27af9fae26b2495ed6f7e354ac58adf7a681
Parents: a1ce97e
Author: Varun Vasudev 
Authored: Tue Aug 2 17:13:20 2016 +0530
Committer: Jian He 
Committed: Tue Apr 18 11:24:18 2017 +0800

--
 .../registry/client/api/RegistryConstants.java  |  20 +++
 .../hadoop/registry/server/dns/RegistryDNS.java |  58 ++-
 .../registry/server/dns/ReverseZoneUtils.java   | 171 +++
 .../registry/server/dns/TestRegistryDNS.java|  31 +++-
 .../server/dns/TestReverseZoneUtils.java|  89 ++
 5 files changed, 361 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c64e27af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index 7115a4c..f4fecfd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -144,6 +144,26 @@ public interface RegistryConstants {
   String KEY_DNS_ZONES_DIR = DNS_PREFIX + "zones-dir";
 
   /**
+   * Split Reverse Zone.
+   * It may be necessary to spit large reverse zone subnets
+   * into multiple zones to handle existing hosts collocated
+   * with containers.
+   */
+  String KEY_DNS_SPLIT_REVERSE_ZONE = DNS_PREFIX + "split-reverse-zone";
+
+  /**
+   * Default value for splitting the reverse zone.
+   */
+  boolean DEFAULT_DNS_SPLIT_REVERSE_ZONE = false;
+
+  /**
+   * Split Reverse Zone IP Range.
+   * How many IPs should be part of each reverse zone split
+   */
+  String KEY_DNS_SPLIT_REVERSE_ZONE_RANGE = DNS_PREFIX +
+  "split-reverse-zone-range";
+
+  /**
* Key to set if the registry is secure: {@value}.
* Turning it on changes the permissions policy from "open access"
* to restrictions on kerberos with the option of

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c64e27af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 52b3c37..126795a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -16,6 +16,7 @@
  */
 package org.apache.hadoop.registry.server.dns;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.commons.net.util.Base64;
@@ -269,17 +270,61 @@ public class RegistryDNS extends AbstractService 
implements DNSOperations,
   }
 
   /**
+   * Return the number of zones in the map.
+   *
+   * @return number of zones in the map
+   */
+  @VisibleForTesting
+  protected int getZoneCount() {
+return zones.size();
+  }
+
+  /**
* Initializes the reverse lookup zone (mapping IP to name).
*
* @param conf the Hadoop configuration.
-   * @throws IOException
+   * @throws IOException if the DNSSEC key can not be read.
*/
   private void initializeReverseLookupZone(Configuration conf)
   throws IOException {
-Name reverseLookupZoneName = getReverseZoneName(conf);
-Zone reverseLookupZone =
-configureZone(reverseLookupZoneName, conf);
-zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
+// Determine if the subnet should be split into
+// multiple reverse zones, this can be necessary in
+// network configurations where the hosts and containers
+

[10/50] [abbrv] hadoop git commit: YARN-5828. Native services client errors out when config formats are uppercase. Contributed by Billie Rinaldi

[jianhe]

YARN-5828. Native services client errors out when config formats are uppercase. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69b454d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69b454d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69b454d8

Branch: refs/heads/yarn-native-services
Commit: 69b454d8afac9d01783498a99a8697027cd11a70
Parents: 99e7eb1
Author: Gour Saha 
Authored: Thu Nov 3 18:15:44 2016 -0700
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../org/apache/slider/core/registry/docstore/ConfigFormat.java   | 4 +++-
 .../src/main/java/org/apache/slider/providers/ProviderUtils.java | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b454d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index ddab606..723b975 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -18,6 +18,8 @@
 
 package org.apache.slider.core.registry.docstore;
 
+import java.util.Locale;
+
 public enum ConfigFormat {
 
   JSON("json"),
@@ -51,7 +53,7 @@ public enum ConfigFormat {
*/
   public static ConfigFormat resolve(String type) {
 for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type)) {
+  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
 return format;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b454d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index c5e6782..39986c1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -611,7 +611,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
   }
   ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
   if (configFormat == null) {
-throw new BadConfigException("Config format " + configFormat +
+throw new BadConfigException("Config format " + configFileType +
 " doesn't exist");
   }
   localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[04/50] [abbrv] hadoop git commit: YARN-5941. Slider handles "per.component" for multiple components incorrectly. Contributed by Billie Rinaldi

[jianhe]

YARN-5941. Slider handles "per.component" for multiple components incorrectly. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6ed5f13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6ed5f13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6ed5f13

Branch: refs/heads/yarn-native-services
Commit: a6ed5f1339547e1ca5705bc8cbde6ff4621a39e5
Parents: 097d254
Author: Gour Saha 
Authored: Wed Nov 30 14:00:22 2016 -0800
Committer: Jian He 
Committed: Tue Apr 18 11:24:17 2017 +0800

--
 .../java/org/apache/slider/api/OptionKeys.java  | 14 ++
 .../org/apache/slider/common/SliderKeys.java|  2 --
 .../apache/slider/providers/ProviderUtils.java  | 20 
 3 files changed, 30 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6ed5f13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 434b1d9..988627d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -53,8 +53,22 @@ public interface OptionKeys extends InternalKeys {
* Prefix for export options: {@value}
*/
   String EXPORT_PREFIX = "export.";
+  /**
+   * Type suffix for config file and package options: {@value}
+   */
   String TYPE_SUFFIX = ".type";
+  /**
+   * Name suffix for config file and package options: {@value}
+   */
   String NAME_SUFFIX = ".name";
+  /**
+   * Per component suffix for config file options: {@value}
+   */
+  String PER_COMPONENT = ".per.component";
+  /**
+   * Per group suffix for config file options: {@value}
+   */
+  String PER_GROUP = ".per.group";
 
   /**
* Zookeeper quorum host list: {@value}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6ed5f13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 3d25d33..adf40ce 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -304,8 +304,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
 
   String APP_RESOURCES = "application.resources";
   String APP_RESOURCES_DIR = "app/resources";
-  String PER_COMPONENT = "per.component";
-  String PER_GROUP = "per.group";
 
   String APP_PACKAGES_DIR = "app/packages";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6ed5f13/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 39986c1..bc237f5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -614,8 +614,16 @@ public class ProviderUtils implements RoleKeys,