hadoop git commit: HADOOP-13768. AliyunOSS: handle the failure in the batch delete operation `deleteDirs`. Contributed by Genmao Yu

[drankye]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 08f93978f -> 5b151290a


HADOOP-13768. AliyunOSS: handle the failure in the batch delete operation 
`deleteDirs`. Contributed by Genmao Yu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b151290
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b151290
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b151290

Branch: refs/heads/trunk
Commit: 5b151290ae2916dc04d6a4338085fcefafa21982
Parents: 08f9397
Author: Kai Zheng 
Authored: Fri Feb 10 14:53:18 2017 +0800
Committer: Kai Zheng 
Committed: Fri Feb 10 14:53:18 2017 +0800

--
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 47 
 .../src/test/resources/contract/aliyun-oss.xml  |  5 +++
 2 files changed, 43 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b151290/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index 9792a78..a944fc1 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -29,6 +29,7 @@ import com.aliyun.oss.model.CompleteMultipartUploadRequest;
 import com.aliyun.oss.model.CompleteMultipartUploadResult;
 import com.aliyun.oss.model.CopyObjectResult;
 import com.aliyun.oss.model.DeleteObjectsRequest;
+import com.aliyun.oss.model.DeleteObjectsResult;
 import com.aliyun.oss.model.GetObjectRequest;
 import com.aliyun.oss.model.InitiateMultipartUploadRequest;
 import com.aliyun.oss.model.InitiateMultipartUploadResult;
@@ -183,14 +184,40 @@ public class AliyunOSSFileSystemStore {
* Delete a list of keys, and update write operation statistics.
*
* @param keysToDelete collection of keys to delete.
+   * @throws IOException if failed to delete objects.
*/
-  public void deleteObjects(List keysToDelete) {
-if (CollectionUtils.isNotEmpty(keysToDelete)) {
-  DeleteObjectsRequest deleteRequest =
-  new DeleteObjectsRequest(bucketName);
-  deleteRequest.setKeys(keysToDelete);
-  ossClient.deleteObjects(deleteRequest);
-  statistics.incrementWriteOps(keysToDelete.size());
+  public void deleteObjects(List keysToDelete) throws IOException {
+if (CollectionUtils.isEmpty(keysToDelete)) {
+  LOG.warn("Keys to delete is empty.");
+  return;
+}
+
+int retry = 10;
+int tries = 0;
+List deleteFailed = keysToDelete;
+while(CollectionUtils.isNotEmpty(deleteFailed)) {
+  DeleteObjectsRequest deleteRequest = new 
DeleteObjectsRequest(bucketName);
+  deleteRequest.setKeys(deleteFailed);
+  // There are two modes to do batch delete:
+  // 1. detail mode: DeleteObjectsResult.getDeletedObjects returns objects
+  // which were deleted successfully.
+  // 2. simple mode: DeleteObjectsResult.getDeletedObjects returns objects
+  // which were deleted unsuccessfully.
+  // Here, we choose the simple mode to do batch delete.
+  deleteRequest.setQuiet(true);
+  DeleteObjectsResult result = ossClient.deleteObjects(deleteRequest);
+  deleteFailed = result.getDeletedObjects();
+  tries++;
+  if (tries == retry) {
+break;
+  }
+}
+
+if (tries == retry && CollectionUtils.isNotEmpty(deleteFailed)) {
+  // Most of time, it is impossible to try 10 times, expect the
+  // Aliyun OSS service problems.
+  throw new IOException("Failed to delete Aliyun OSS objects for " +
+  tries + " times.");
 }
   }
 
@@ -198,8 +225,9 @@ public class AliyunOSSFileSystemStore {
* Delete a directory from Aliyun OSS.
*
* @param key directory key to delete.
+   * @throws IOException if failed to delete directory.
*/
-  public void deleteDirs(String key) {
+  public void deleteDirs(String key) throws IOException {
 key = AliyunOSSUtils.maybeAddTrailingSlash(key);
 ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
 listRequest.setPrefix(key);
@@ -496,8 +524,9 @@ public class AliyunOSSFileSystemStore {
* Clean up all objects matching the prefix.
*
* @param prefix Aliyun OSS object prefix.
+   * @throws IOException if failed to clean up objects.
*/
-  public void purge(String prefix) {
+  public void purge(String prefix) throws IOException {
 String key;
 try {
   ObjectListing 

hadoop git commit: YARN-6151. FS preemption does not consider child queues over fairshare if the parent is under. (Yufei Gu via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 6b3443fbf -> 5d0ec2e24


YARN-6151. FS preemption does not consider child queues over fairshare if the 
parent is under. (Yufei Gu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d0ec2e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d0ec2e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d0ec2e2

Branch: refs/heads/branch-2.8
Commit: 5d0ec2e24dac40767d521408dfd3a5b51f88c541
Parents: 6b3443f
Author: Karthik Kambatla 
Authored: Thu Feb 9 20:23:50 2017 -0800
Committer: Karthik Kambatla 
Committed: Thu Feb 9 20:23:50 2017 -0800

--
 .../scheduler/fair/FSLeafQueue.java | 12 +---
 .../scheduler/fair/FSParentQueue.java   | 10 +++
 .../resourcemanager/scheduler/fair/FSQueue.java | 24 +++
 .../scheduler/fair/TestFairScheduler.java   |  4 +-
 .../fair/TestFairSchedulerPreemption.java   | 68 +++-
 5 files changed, 102 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d0ec2e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 51849f8..87a5448 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -351,7 +351,7 @@ public class FSLeafQueue extends FSQueue {
 RMContainer toBePreempted = null;
 
 // If this queue is not over its fair share, reject
-if (!preemptContainerPreCheck()) {
+if (!canBePreempted()) {
   return toBePreempted;
 }
 
@@ -534,16 +534,6 @@ public class FSLeafQueue extends FSQueue {
   }
 
   /**
-   * Helper method to check if the queue should preempt containers
-   *
-   * @return true if check passes (can preempt) or false otherwise
-   */
-  private boolean preemptContainerPreCheck() {
-return parent.getPolicy().checkIfUsageOverFairShare(getResourceUsage(),
-getFairShare());
-  }
-
-  /**
* Is a queue being starved for its min share.
*/
   @VisibleForTesting

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d0ec2e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index fe0e3e2..e08b386 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -260,6 +260,16 @@ public class FSParentQueue extends FSQueue {
 readLock.lock();
 try {
   for (FSQueue queue : childQueues) {
+// Skip selection for non-preemptable queue
+if (!queue.canBePreempted()) {
+  if (LOG.isDebugEnabled()) {
+LOG.debug("skipping from queue=" + getName()
++ " because it's a non-preemptable queue or there is no"
++ " sub-queues whose resource usage exceeds fair share.");
+  }
+  continue;
+}
+
 if (candidateQueue == null ||
 comparator.compare(queue, candidateQueue) > 0) {
   candidateQueue = queue;

[48/50] [abbrv] hadoop git commit: YARN-6132. SliderClient bondToCluster should call findInstance with live states filter. Contributed by Gour Saha

[jianhe]

YARN-6132. SliderClient bondToCluster should call findInstance with live states 
filter. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/585fc04e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/585fc04e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/585fc04e

Branch: refs/heads/yarn-native-services
Commit: 585fc04e6f9e614f0bfb24acd1d7745162901218
Parents: 37cb3c5
Author: Billie Rinaldi 
Authored: Mon Jan 30 09:20:57 2017 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../src/main/java/org/apache/slider/client/SliderClient.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/585fc04e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 032eb6e..e84809a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -3519,7 +3519,8 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 if (clustername == null) {
   throw unknownClusterException("(undefined)");
 }
-ApplicationReport instance = findInstance(clustername);
+ApplicationReport instance = findInstance(clustername,
+SliderUtils.getAllLiveAppStates());
 if (null == instance) {
   throw unknownClusterException(clustername);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[45/50] [abbrv] hadoop git commit: YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a06f940/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
deleted file mode 100644
index e82ad84..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ /dev/null
@@ -1,256 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.services.security;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.core.conf.MapOperations;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-//import java.nio.file.Files;
-//import java.nio.file.Path;
-//import java.nio.file.Paths;
-//import java.nio.file.attribute.PosixFilePermission;
-//import java.nio.file.attribute.PosixFilePermissions;
-
-
-/**
- *
- */
-public class SecurityUtils {
-  private static final Logger LOG =
-  LoggerFactory.getLogger(SecurityUtils.class);
-
-  private static String CA_CONFIG_CONTENTS =  "HOME= .\n"
-+ "RANDFILE= 
$ENV::HOME/.rnd\n\n"
-+ "[ ca ]\n"
-+ "default_ca = 
CA_CLIENT\n"
-+ "[ CA_CLIENT ]\n"
-+ "dir= 
${SEC_DIR}/db\n"
-+ "certs  = 
$dir/certs\n"
-+ "new_certs_dir  = 
$dir/newcerts\n"
-+ "\n"
-+ "database   = 
$dir/index.txt\n"
-+ "serial = 
$dir/serial\n"
-+ "default_days   = 365
\n"
-+ "\n"
-+ "default_crl_days   = 7  \n"
-+ "default_md = sha256 
\n"
-+ "\n"
-+ "policy = 
policy_anything \n"
-+ "\n"
-+ "[ policy_anything ]\n"
-+ "countryName= 
optional\n"
-+ "stateOrProvinceName= 
optional\n"
-+ "localityName   = 
optional\n"
-+ "organizationName   = 
optional\n"
-+ "organizationalUnitName = 
optional\n"
-+ "commonName = 
optional\n"
-+ "emailAddress   = 
optional\n"
-+ "\n"
-+ "[req]\n"
-+ "distinguished_name = 
req_distinguished_name\n"
-+ 

[41/50] [abbrv] hadoop git commit: YARN-4757. Add the ability to split reverse zone subnets. Contributed by Shane Kumpf.

[jianhe]

YARN-4757. Add the ability to split reverse zone subnets. Contributed by Shane 
Kumpf.

(cherry picked from commit 9bff70f1316d889c3109c68dd308c8f456229bf8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d892feef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d892feef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d892feef

Branch: refs/heads/yarn-native-services
Commit: d892feefe3b6aa0a3b43710b4019045dff077549
Parents: 8ba1de8
Author: Varun Vasudev 
Authored: Tue Aug 2 17:13:20 2016 +0530
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../registry/client/api/RegistryConstants.java  |  20 +++
 .../hadoop/registry/server/dns/RegistryDNS.java |  58 ++-
 .../registry/server/dns/ReverseZoneUtils.java   | 171 +++
 .../registry/server/dns/TestRegistryDNS.java|  31 +++-
 .../server/dns/TestReverseZoneUtils.java|  89 ++
 5 files changed, 361 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892feef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
index 7115a4c..f4fecfd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/RegistryConstants.java
@@ -144,6 +144,26 @@ public interface RegistryConstants {
   String KEY_DNS_ZONES_DIR = DNS_PREFIX + "zones-dir";
 
   /**
+   * Split Reverse Zone.
+   * It may be necessary to spit large reverse zone subnets
+   * into multiple zones to handle existing hosts collocated
+   * with containers.
+   */
+  String KEY_DNS_SPLIT_REVERSE_ZONE = DNS_PREFIX + "split-reverse-zone";
+
+  /**
+   * Default value for splitting the reverse zone.
+   */
+  boolean DEFAULT_DNS_SPLIT_REVERSE_ZONE = false;
+
+  /**
+   * Split Reverse Zone IP Range.
+   * How many IPs should be part of each reverse zone split
+   */
+  String KEY_DNS_SPLIT_REVERSE_ZONE_RANGE = DNS_PREFIX +
+  "split-reverse-zone-range";
+
+  /**
* Key to set if the registry is secure: {@value}.
* Turning it on changes the permissions policy from "open access"
* to restrictions on kerberos with the option of

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d892feef/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index 52b3c37..126795a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -16,6 +16,7 @@
  */
 package org.apache.hadoop.registry.server.dns;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.filefilter.IOFileFilter;
 import org.apache.commons.net.util.Base64;
@@ -269,17 +270,61 @@ public class RegistryDNS extends AbstractService 
implements DNSOperations,
   }
 
   /**
+   * Return the number of zones in the map.
+   *
+   * @return number of zones in the map
+   */
+  @VisibleForTesting
+  protected int getZoneCount() {
+return zones.size();
+  }
+
+  /**
* Initializes the reverse lookup zone (mapping IP to name).
*
* @param conf the Hadoop configuration.
-   * @throws IOException
+   * @throws IOException if the DNSSEC key can not be read.
*/
   private void initializeReverseLookupZone(Configuration conf)
   throws IOException {
-Name reverseLookupZoneName = getReverseZoneName(conf);
-Zone reverseLookupZone =
-configureZone(reverseLookupZoneName, conf);
-zones.put(reverseLookupZone.getOrigin(), reverseLookupZone);
+// Determine if the subnet should be split into
+// multiple reverse zones, this can be necessary in
+// network configurations where the hosts and containers
+

[50/50] [abbrv] hadoop git commit: YARN-5993. Allow native services quicklinks to be exported for each component. Contributed by Billie Rinaldi

[jianhe]

YARN-5993. Allow native services quicklinks to be exported for each component. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa73e1d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa73e1d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa73e1d5

Branch: refs/heads/yarn-native-services
Commit: fa73e1d52a9add765dbb2893b41fceda995e0342
Parents: d892fee
Author: Gour Saha 
Authored: Wed Dec 28 15:24:36 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../apache/slider/common/tools/SliderUtils.java |   8 +-
 .../core/registry/docstore/ExportEntry.java |  20 +++
 .../registry/docstore/PublishedExports.java |  23 ++-
 .../apache/slider/providers/ProviderUtils.java  | 133 
 .../providers/docker/DockerProviderService.java | 151 ---
 .../server/appmaster/web/view/IndexBlock.java   |  15 +-
 6 files changed, 177 insertions(+), 173 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa73e1d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index a107dfb..20c7831 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -183,17 +183,17 @@ public final class SliderUtils {
 return !isUnset(s);
   }
 
-  public static boolean isEmpty(List l) {
+  public static boolean isEmpty(Collection l) {
 return l == null || l.isEmpty();
   }
 
   /**
-   * Probe for a list existing and not being empty
-   * @param l list
+   * Probe for a collection existing and not being empty
+   * @param l collection
* @return true if the reference is valid and it contains entries
*/
 
-  public static boolean isNotEmpty(List l) {
+  public static boolean isNotEmpty(Collection l) {
 return l != null && !l.isEmpty();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa73e1d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
index 4bcf6c1..dd6e034 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ExportEntry.java
@@ -107,6 +107,26 @@ public class ExportEntry {
   }
 
   @Override
+  public boolean equals(Object o) {
+if (this == o) return true;
+if (o == null || getClass() != o.getClass()) return false;
+
+ExportEntry that = (ExportEntry) o;
+
+if (value != null ? !value.equals(that.value) : that.value != null)
+  return false;
+return containerId != null ? containerId.equals(that.containerId) :
+that.containerId == null;
+  }
+
+  @Override
+  public int hashCode() {
+int result = value != null ? value.hashCode() : 0;
+result = 31 * result + (containerId != null ? containerId.hashCode() : 0);
+return result;
+  }
+
+  @Override
   public String toString() {
 return new StringBuilder("ExportEntry{").
 append("value='").append(value).append("',").

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa73e1d5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java

[19/50] [abbrv] hadoop git commit: YARN-5813. Slider should not try to set a negative lifetime timeout value. Contributed by Jian He

[jianhe]

YARN-5813. Slider should not try to set a negative lifetime timeout value. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a47af4d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a47af4d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a47af4d4

Branch: refs/heads/yarn-native-services
Commit: a47af4d41e50c10f1a213697f2982189ebf91781
Parents: 35431ea
Author: Gour Saha 
Authored: Tue Nov 1 17:39:54 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../src/main/java/org/apache/slider/client/SliderClient.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a47af4d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index d1f88c5..ea10ed0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2120,7 +2120,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 amLauncher.setKeepContainersOverRestarts(true);
 // set lifetime in submission context;
 Map appTimeout = new HashMap<>();
-appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+if (lifetime >= 0) {
+  appTimeout.put(ApplicationTimeoutType.LIFETIME, lifetime);
+}
 amLauncher.submissionContext.setApplicationTimeouts(appTimeout);
 int maxAppAttempts = config.getInt(KEY_AM_RESTART_LIMIT, 0);
 amLauncher.setMaxAppAttempts(maxAppAttempts);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[30/50] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

YARN-5909. Remove agent related code in slider AM. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf5ffbb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf5ffbb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf5ffbb9

Branch: refs/heads/yarn-native-services
Commit: cf5ffbb9a5403e93f5e798f23b76757c97db0e05
Parents: c8c50e1
Author: Jian He 
Authored: Tue Nov 29 13:06:45 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 hadoop-project/pom.xml  |5 -
 .../services/webapp/ApplicationApiWebApp.java   |2 +-
 .../hadoop-yarn-slider-core/pom.xml |   23 +-
 .../apache/slider/common/tools/SliderUtils.java |1 -
 .../providers/AbstractProviderService.java  |   21 +-
 .../slider/providers/ProviderService.java   |9 +-
 .../providers/agent/AgentClientProvider.java|  713 -
 .../providers/agent/AgentLaunchParameter.java   |  130 -
 .../providers/agent/AgentProviderFactory.java   |   47 -
 .../providers/agent/AgentProviderService.java   | 2850 --
 .../slider/providers/agent/AgentRoles.java  |   38 -
 .../slider/providers/agent/AgentUtils.java  |  150 -
 .../apache/slider/providers/agent/Command.java  |   59 -
 .../slider/providers/agent/CommandResult.java   |   40 -
 .../providers/agent/ComponentCommandOrder.java  |  225 --
 .../providers/agent/ComponentInstanceState.java |  340 ---
 .../providers/agent/ComponentTagProvider.java   |  127 -
 .../slider/providers/agent/ContainerState.java  |   41 -
 .../providers/agent/HeartbeatMonitor.java   |  130 -
 .../apache/slider/providers/agent/State.java|  199 --
 .../application/metadata/AbstractComponent.java |   80 -
 .../metadata/AbstractMetainfoParser.java|  130 -
 .../metadata/AbstractMetainfoSchema.java|   69 -
 .../metadata/AddonPackageMetainfoParser.java|   53 -
 .../agent/application/metadata/Application.java |  193 --
 .../metadata/ApplicationPackage.java|   69 -
 .../application/metadata/CommandOrder.java  |   61 -
 .../application/metadata/CommandScript.java |   72 -
 .../agent/application/metadata/Component.java   |  217 --
 .../application/metadata/ComponentCommand.java  |   85 -
 .../application/metadata/ComponentExport.java   |   54 -
 .../metadata/ComponentsInAddonPackage.java  |   26 -
 .../agent/application/metadata/ConfigFile.java  |   59 -
 .../application/metadata/DefaultConfig.java |   39 -
 .../metadata/DefaultConfigParser.java   |   54 -
 .../application/metadata/DockerContainer.java   |  187 --
 .../metadata/DockerContainerInputFile.java  |   50 -
 .../metadata/DockerContainerMount.java  |   60 -
 .../metadata/DockerContainerPort.java   |   66 -
 .../agent/application/metadata/Export.java  |   61 -
 .../agent/application/metadata/ExportGroup.java |   71 -
 .../agent/application/metadata/Metainfo.java|  118 -
 .../application/metadata/MetainfoParser.java|   97 -
 .../agent/application/metadata/OSPackage.java   |   51 -
 .../agent/application/metadata/OSSpecific.java  |   57 -
 .../agent/application/metadata/Package.java |   60 -
 .../application/metadata/PropertyInfo.java  |   54 -
 .../agent/application/metadata/Validate.java|   27 -
 .../org/apache/slider/providers/agent/todo.md   |   22 -
 .../server/appmaster/SliderAppMaster.java   |  140 +-
 .../server/appmaster/web/AgentService.java  |   37 -
 .../slider/server/appmaster/web/WebAppApi.java  |7 -
 .../server/appmaster/web/WebAppApiImpl.java |6 -
 .../appmaster/web/rest/AMWebServices.java   |1 -
 .../web/rest/agent/AgentCommandType.java|   23 -
 .../appmaster/web/rest/agent/AgentEnv.java  |  376 ---
 .../appmaster/web/rest/agent/AgentResource.java |  118 -
 .../web/rest/agent/AgentRestOperations.java |   28 -
 .../appmaster/web/rest/agent/AgentWebApp.java   |  258 --
 .../web/rest/agent/AgentWebServices.java|   40 -
 .../appmaster/web/rest/agent/CommandReport.java |  207 --
 .../web/rest/agent/ComponentStatus.java |  129 -
 .../appmaster/web/rest/agent/DiskInfo.java  |  128 -
 .../web/rest/agent/ExecutionCommand.java|  310 --
 .../appmaster/web/rest/agent/HeartBeat.java |  149 -
 .../web/rest/agent/HeartBeatResponse.java   |  147 -
 .../appmaster/web/rest/agent/HostInfo.java  |  398 ---
 .../appmaster/web/rest/agent/HostStatus.java|   63 -
 .../appmaster/web/rest/agent/Register.java  |  193 --
 .../web/rest/agent/RegistrationCommand.java |   43 -
 .../web/rest/agent/RegistrationResponse.java|  133 -
 .../web/rest/agent/RegistrationStatus.java  |   22 -
 .../appmaster/web/rest/agent/StatusCommand.java |  152 -
 .../agent/TestAgentClientProvider.java  

[21/50] [abbrv] hadoop git commit: YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ba1de8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
new file mode 100644
index 000..52b3c37
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -0,0 +1,1534 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.IOFileFilter;
+import org.apache.commons.net.util.Base64;
+import org.apache.commons.net.util.SubnetUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.registry.client.api.DNSOperations;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.DNSKEYRecord;
+import org.xbill.DNS.DNSSEC;
+import org.xbill.DNS.DSRecord;
+import org.xbill.DNS.ExtendedFlags;
+import org.xbill.DNS.Flags;
+import org.xbill.DNS.Header;
+import org.xbill.DNS.Message;
+import org.xbill.DNS.NSRecord;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.NameTooLongException;
+import org.xbill.DNS.OPTRecord;
+import org.xbill.DNS.Opcode;
+import org.xbill.DNS.RRSIGRecord;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Rcode;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SOARecord;
+import org.xbill.DNS.Section;
+import org.xbill.DNS.SetResponse;
+import org.xbill.DNS.TSIG;
+import org.xbill.DNS.TSIGRecord;
+import org.xbill.DNS.TextParseException;
+import org.xbill.DNS.Type;
+import org.xbill.DNS.Zone;
+
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.DatagramChannel;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.security.KeyFactory;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivateKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.RSAPrivateKeySpec;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
+
+/**
+ * A DNS service reflecting the state of the YARN registry.  Records are 
created
+ * based on service records available in the YARN ZK-based registry.
+ */
+public class RegistryDNS extends AbstractService implements DNSOperations,
+ZoneSelector {
+
+  public static final String CONTAINER = "container";
+
+  static final int FLAG_DNSSECOK = 1;
+  static final int FLAG_SIGONLY = 2;
+
+  private static final Logger LOG =
+  

[18/50] [abbrv] hadoop git commit: YARN-5796. Convert enums values in service code to upper case and special handling of an error. Contributed by Gour Saha

[jianhe]

YARN-5796. Convert enums values in service code to upper case and special 
handling of an error. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35431eaa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35431eaa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35431eaa

Branch: refs/heads/yarn-native-services
Commit: 35431eaa4bedcb9288aa1169fb117f135ff7fcc9
Parents: 2ca2e62
Author: Jian He 
Authored: Tue Nov 1 11:00:11 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../yarn/services/api/impl/ApplicationApiService.java |  4 
 .../apache/hadoop/yarn/services/resource/Artifact.java|  2 +-
 .../apache/hadoop/yarn/services/resource/ConfigFile.java  |  4 ++--
 .../hadoop/yarn/services/resource/ReadinessCheck.java |  2 +-
 .../hadoop/yarn/services/utils/RestApiErrorMessages.java  | 10 +++---
 5 files changed, 15 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/35431eaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index cf43ac2..37bd134 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -172,6 +172,10 @@ public class ApplicationApiService implements 
ApplicationApi {
 applicationStatus.setDiagnostics(ERROR_APPLICATION_IN_USE);
 return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
 .build();
+  } else if (se.getExitCode() == SliderExitCodes.EXIT_INSTANCE_EXISTS) {
+applicationStatus.setDiagnostics(ERROR_APPLICATION_INSTANCE_EXISTS);
+return Response.status(Status.BAD_REQUEST).entity(applicationStatus)
+.build();
   } else {
 applicationStatus.setDiagnostics(se.getMessage());
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/35431eaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
index 9ac2bc7..87fcf89 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Artifact.java
@@ -40,7 +40,7 @@ public class Artifact implements Serializable {
   private String id = null;
 
   public enum TypeEnum {
-DOCKER("docker"), TARBALL("tarball"), APPLICATION("application");
+DOCKER("DOCKER"), TARBALL("TARBALL"), APPLICATION("APPLICATION");
 
 private String value;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/35431eaa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
index 3ced153..01d976f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ConfigFile.java
+++ 

[06/50] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58ad73fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
new file mode 100644
index 000..cfcae95
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.slider.providers.PlacementPolicy;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+/**
+ * An Application resource has the following attributes.
+ **/
+
+@ApiModel(description = "An Application resource has the following 
attributes.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonPropertyOrder({ " name, state, resource, numberOfContainers, lifetime, 
containers " })
+public class Application extends BaseResource {
+  private static final long serialVersionUID = -4491694636566094885L;
+
+  private String id = null;
+  private String name = null;
+  private Artifact artifact = null;
+  private Resource resource = null;
+  private String launchCommand = null;
+  private Date launchTime = null;
+  private Long numberOfContainers = null;
+  private Long numberOfRunningContainers = null;
+  private Long lifetime = null;
+  private PlacementPolicy placementPolicy = null;
+  private List components = null;
+  private Configuration configuration = null;
+  private List containers = new ArrayList<>();
+  private ApplicationState state = null;
+  private Map quicklinks = null;
+  private String queue;
+
+  /**
+   * A unique application id.
+   **/
+  public Application id(String id) {
+this.id = id;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", required = true, value = "A unique 
application id.")
+  @JsonProperty("id")
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  /**
+   * A unique application name.
+   **/
+  public Application name(String name) {
+this.name = name;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", required = true, value = "A unique 
application name.")
+  @JsonProperty("name")
+  public String getName() {
+return name;
+  }
+
+  public void setName(String name) {
+this.name = name;
+  }
+
+  /**
+   * Artifact of single-component applications. Mandatory if components
+   * attribute is not specified.
+   **/
+  public Application artifact(Artifact artifact) {
+this.artifact = artifact;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Artifact of single-component 
applications. Mandatory if components attribute is not specified.")
+  @JsonProperty("artifact")
+  public Artifact getArtifact() {
+return artifact;
+  }
+
+  public void setArtifact(Artifact artifact) {
+this.artifact = artifact;
+  }
+
+  /**
+   * Resource of single-component applications or the global default for
+   * multi-component applications. Mandatory if it is a single-component
+   * application and if cpus and memory are not specified at the 

[37/50] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

YARN-5961. Generate native services protobuf classes during build. Contributed 
by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cc72ab2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cc72ab2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cc72ab2

Branch: refs/heads/yarn-native-services
Commit: 8cc72ab26bfc268b9454ef35c68e8f0af23ec65d
Parents: 015add1
Author: Jian He 
Authored: Sat Dec 3 13:27:04 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../hadoop-yarn-slider-core/pom.xml |65 +-
 .../org/apache/slider/api/proto/Messages.java   | 34473 -
 .../slider/api/proto/SliderClusterAPI.java  |  2293 --
 3 files changed, 27 insertions(+), 36804 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc72ab2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 10cf6b1..7453d12 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -37,6 +37,33 @@
 
 
   
+org.apache.hadoop
+hadoop-maven-plugins
+
+  
+compile-protoc
+
+  protoc
+
+
+  ${protobuf.version}
+  ${protoc.path}
+  
+${basedir}/src/main/proto
+  
+  
+${basedir}/src/main/proto
+
+  SliderClusterMessages.proto
+  SliderClusterProtocol.proto
+
+  
+
+  
+
+  
+
+  
 org.apache.maven.plugins
 maven-jar-plugin
 
@@ -85,9 +112,6 @@
   
 **/*.json
 src/main/resources/webapps/slideram/.keep
-
-
src/main/java/org/apache/slider/api/proto/Messages.java
-
src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
   
 
   
@@ -362,41 +386,6 @@
 
   
 
-
-  compile-protobuf
-  
-
-  
-org.apache.hadoop
-hadoop-maven-plugins
-
-  
-compile-protoc
-generate-sources
-
-  protoc
-
-
-  ${protobuf.version}
-  protoc
-  
-${basedir}/src/main/proto
-  
-  
-${basedir}/src/main/proto
-
-  SliderClusterMessages.proto
-  SliderClusterProtocol.proto
-
-  
-  ${basedir}/src/main/java
-
-  
-
-  
-
-  
-
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[32/50] [abbrv] hadoop git commit: YARN-5943. Write native services container stderr file to log directory. Contributed by Billie Rinaldi

[jianhe]

YARN-5943. Write native services container stderr file to log directory. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a52d737
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a52d737
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a52d737

Branch: refs/heads/yarn-native-services
Commit: 9a52d73708deb6dc2e0597a7c5aeefba30839559
Parents: 528626b
Author: Gour Saha 
Authored: Wed Nov 30 10:30:39 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/providers/docker/DockerProviderService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a52d737/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index cc319ee..e4a7cdf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -193,7 +193,8 @@ public class DockerProviderService extends 
AbstractProviderService implements
 "/bin/bash"));
 
 operation.add("> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
-+ OUT_FILE + " 2>" + ERR_FILE);
++ OUT_FILE + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/"
++ ERR_FILE);
 
 launcher.addCommand(operation.build());
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[04/50] [abbrv] hadoop git commit: YARN-5623. Apply SLIDER-1166 to yarn-native-services branch. Contributed by Gour Saha

[jianhe]

YARN-5623. Apply SLIDER-1166 to yarn-native-services branch. Contributed by 
Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/742569f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/742569f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/742569f0

Branch: refs/heads/yarn-native-services
Commit: 742569f08e83fb6693f8b3fbe28240709579059e
Parents: bb86096
Author: Jian He 
Authored: Mon Sep 12 10:44:39 2016 +0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/client/SliderClient.java  |  9 --
 .../apache/slider/core/zk/ZKIntegration.java| 34 +---
 2 files changed, 36 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/742569f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 5096bb7..fe4f1d2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -604,11 +604,14 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   BlockingZKWatcher watcher = new BlockingZKWatcher();
   client = ZKIntegration.newInstance(registryQuorum, user, clusterName, 
true, false, watcher,
   ZKIntegration.SESSION_TIMEOUT);
-  client.init();
-  watcher.waitForZKConnection(2 * 1000);
+  boolean fromCache = client.init();
+  if (!fromCache) {
+watcher.waitForZKConnection(2 * 1000);
+  }
 } catch (InterruptedException e) {
   client = null;
-  log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
+  log.warn("Interrupted - unable to connect to zookeeper quorum {}",
+  registryQuorum, e);
 } catch (IOException e) {
   log.warn("Unable to connect to zookeeper quorum {}", registryQuorum, e);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/742569f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index ca41e4b..6ed58d5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -33,6 +33,8 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 
@@ -65,6 +67,8 @@ public class ZKIntegration implements Watcher, Closeable {
   private final String clustername;
   private final String userPath;
   private int sessionTimeout = SESSION_TIMEOUT;
+  private static final Map ZK_SESSIONS = new HashMap<>();
+
 /**
  flag to set to indicate that the user path should be created if
  it is not already there
@@ -93,10 +97,32 @@ public class ZKIntegration implements Watcher, Closeable {
 this.userPath = mkSliderUserPath(username);
   }
 
-  public void init() throws IOException {
-assert zookeeper == null;
-log.debug("Binding ZK client to {}", zkConnection);
-zookeeper = new ZooKeeper(zkConnection, sessionTimeout, this, 
canBeReadOnly);
+  /**
+   * Returns true only if an active ZK session is available and retrieved from
+   * cache, false when it has to create a new one.
+   *
+   * @return true if from cache, false when new session created
+   * @throws IOException
+   */
+  public synchronized 

[08/50] [abbrv] hadoop git commit: YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha

[jianhe]

YARN-5675. Swagger definition for YARN service API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b47fb5d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b47fb5d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b47fb5d3

Branch: refs/heads/yarn-native-services
Commit: b47fb5d395330b1c8d4e6e7a2d8bfe6873588614
Parents: 58ad73f
Author: Jian He 
Authored: Wed Oct 12 13:27:53 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 416 +++
 1 file changed, 416 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b47fb5d3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
new file mode 100644
index 000..6169fcd
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -0,0 +1,416 @@
+# Hadoop YARN REST APIs for services v1 spec in YAML
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+swagger: '2.0'
+info:
+  title: "[YARN-4793] Simplified API layer for services and beyond"
+  description: |
+Bringing a new service on YARN today is not a simple experience. The APIs 
of existing frameworks are either too low level (native YARN), require writing 
new code (for frameworks with programmatic APIs) or writing a complex spec (for 
declarative frameworks). In addition to building critical building blocks 
inside YARN (as part of other efforts at 
link:https://issues.apache.org/jira/browse/YARN-4692[YARN-4692]), there is a 
need for simplifying the user facing story for building services. Experience of 
projects like Apache Slider running real-life services like HBase, Storm, 
Accumulo, Solr etc, gives us some very good insights on how simplified APIs for 
services should look like.
+
+
+To this end, we should look at a new simple-services API layer backed by 
REST interfaces. This API can be used to create and manage the lifecycle of 
YARN services. Services here can range from simple single-component apps to 
complex multi-component assemblies needing orchestration.
+
+
+We should also look at making this a unified REST based entry point for 
other important features like resource-profile management 
(link:https://issues.apache.org/jira/browse/YARN-3926[YARN-3926]), 
package-definitions' lifecycle-management and service-discovery 
(link:https://issues.apache.org/jira/browse/YARN-913[YARN-913]/link:https://issues.apache.org/jira/browse/YARN-4757[YARN-4757]).
 We also need to flesh out its relation to our present much lower level REST 
APIs (link:https://issues.apache.org/jira/browse/YARN-1695[YARN-1695]) in YARN 
for application-submission and management.
+
+
+This document spotlights on this specification. In most of the cases, the 
application owner will not be forced to make any changes to their application. 
This is primarily true if the application is packaged with containerization 
technologies like docker. Irrespective of how complex the application is, there 
will be hooks provided at appropriate layers to allow pluggable and 
customizable application behavior.
+
+  version: "1.0.0"
+  license:
+name: Apache 2.0
+url: http://www.apache.org/licenses/LICENSE-2.0.html
+# the domain of the service
+host: host.mycompany.com
+# array of all schemes that your API supports
+schemes:
+  - http
+  - https
+# will be prefixed to all paths

[42/50] [abbrv] hadoop git commit: YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f380fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
index c8b3adb..1af883e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/MiniZooKeeperCluster.java
@@ -137,8 +137,6 @@ public class MiniZooKeeperCluster extends AbstractService {
   }
 
   /**
-   * @param baseDir
-   * @param numZooKeeperServers
* @return ClientPort server bound to, -1 if there was a
* binding problem and we couldn't pick another port.
* @throws IOException
@@ -229,17 +227,6 @@ public class MiniZooKeeperCluster extends AbstractService {
 }
   }
 
-  /**
-   * Delete the basedir
-   */
-  private void deleteBaseDir() {
-if (baseDir != null) {
-  baseDir.delete();
-  baseDir = null;
-}
-
-  }
-
   @Override
   protected void serviceStop() throws Exception {
 
@@ -359,7 +346,7 @@ public class MiniZooKeeperCluster extends AbstractService {
 try {
   sock = new Socket("localhost", port);
   OutputStream outstream = sock.getOutputStream();
-  outstream.write("stat".getBytes());
+  outstream.write("stat".getBytes("UTF-8"));
   outstream.flush();
 } finally {
   IOUtils.closeSocket(sock);
@@ -387,10 +374,10 @@ public class MiniZooKeeperCluster extends AbstractService 
{
 BufferedReader reader = null;
 try {
   OutputStream outstream = sock.getOutputStream();
-  outstream.write("stat".getBytes());
+  outstream.write("stat".getBytes("UTF-8"));
   outstream.flush();
 
-  Reader isr = new InputStreamReader(sock.getInputStream());
+  Reader isr = new InputStreamReader(sock.getInputStream(), "UTF-8");
   reader = new BufferedReader(isr);
   String line = reader.readLine();
   if (line != null && line.startsWith("Zookeeper version:")) {
@@ -412,12 +399,4 @@ public class MiniZooKeeperCluster extends AbstractService {
 }
 return false;
   }
-
-  public int getClientPort() {
-return clientPort;
-  }
-
-  public String getZkQuorum() {
-return zkQuorum;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f380fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 6ed58d5..ef96c9b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -43,14 +43,14 @@ public class ZKIntegration implements Watcher, Closeable {
 /**
  * Base path for services
  */
-  public static String ZK_SERVICES = "services";
+  public static final String ZK_SERVICES = "services";
   /**
* Base path for all Slider references
*/
-  public static String ZK_SLIDER = "slider";
-  public static String ZK_USERS = "users";
-  public static String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
-  public static String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
+  public static final String ZK_SLIDER = "slider";
+  public static final String ZK_USERS = "users";
+  public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
+  public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
 
   public static final List ZK_USERS_PATH_LIST = new 
ArrayList();
   static {
@@ -59,7 +59,7 @@ public class ZKIntegration implements Watcher, Closeable {
 ZK_USERS_PATH_LIST.add(ZK_USERS);
   }
 
-  public static int SESSION_TIMEOUT = 3;
+  public static final int SESSION_TIMEOUT = 3;
   

[46/50] [abbrv] hadoop git commit: YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He

[jianhe]

YARN-5975. Remove the agent - slider AM ssl related code. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a06f940
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a06f940
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a06f940

Branch: refs/heads/yarn-native-services
Commit: 7a06f9405392c1025acd801cdc124ea87b762392
Parents: 8cc72ab
Author: Billie Rinaldi 
Authored: Tue Dec 13 10:16:09 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../slider/api/SliderClusterProtocol.java   |   3 -
 .../slider/api/proto/RestTypeMarshalling.java   |  36 --
 .../org/apache/slider/client/SliderClient.java  |  80 ---
 .../client/ipc/SliderClusterOperations.java |  28 +-
 .../slider/common/params/ActionClientArgs.java  |  31 +-
 .../apache/slider/providers/ProviderUtils.java  | 141 -
 .../providers/docker/DockerProviderService.java |   5 -
 .../server/appmaster/SliderAppMaster.java   |  18 +-
 .../rpc/SliderClusterProtocolPBImpl.java|  12 -
 .../rpc/SliderClusterProtocolProxy.java |  18 +-
 .../server/appmaster/rpc/SliderIPCService.java  |  54 +-
 .../slider/server/appmaster/web/WebAppApi.java  |  23 -
 .../server/appmaster/web/WebAppApiImpl.java |  37 +-
 .../AbstractSecurityStoreGenerator.java |  98 
 .../services/security/CertificateManager.java   | 495 -
 .../services/security/KeystoreGenerator.java|  64 ---
 .../server/services/security/SecurityStore.java |  66 ---
 .../security/SecurityStoreGenerator.java|  40 --
 .../server/services/security/SecurityUtils.java | 256 -
 .../services/security/SignCertResponse.java |  67 ---
 .../server/services/security/SignMessage.java   |  54 --
 .../services/security/StoresGenerator.java  |  68 ---
 .../services/security/TruststoreGenerator.java  |  62 ---
 .../src/main/proto/SliderClusterProtocol.proto  |   6 -
 .../security/TestCertificateManager.java| 540 ---
 .../TestMultiThreadedStoreGeneration.java   | 156 --
 26 files changed, 14 insertions(+), 2444 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a06f940/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 33fce22..893e706 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -173,7 +173,4 @@ public interface SliderClusterProtocol extends 
VersionedProtocol {
   Messages.WrappedJsonProto 
getModelResolvedResources(Messages.EmptyPayloadProto request) throws 
IOException;
 
   Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto 
request) throws IOException;
-
-  Messages.GetCertificateStoreResponseProto 
getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
-  throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a06f940/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index 17fd965..ec35028 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -18,8 +18,6 @@
 
 package org.apache.slider.api.proto;
 
-import com.google.protobuf.ByteString;
-import 

[25/50] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
deleted file mode 100644
index 0150079..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/Register.java
+++ /dev/null
@@ -1,193 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.server.appmaster.web.rest.agent;
-
-import org.apache.slider.providers.agent.State;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.annotate.JsonProperty;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-import java.util.Map;
-
-/** Data model for agent to send heartbeat to ambari and/or app master. */
-@JsonIgnoreProperties(ignoreUnknown = true)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-public class Register {
-  private int responseId = -1;
-  private long timestamp;
-  private String label;
-  private int currentPingPort;
-  private HostInfo hardwareProfile;
-  private String publicHostname;
-  private String tags;
-  private AgentEnv agentEnv;
-  private String agentVersion;
-  private State actualState;
-  private State expectedState;
-  private Map allocatedPorts;
-  private Map logFolders;
-  private String pkg;
-  private String appVersion;
-
-  @JsonProperty("responseId")
-  public int getResponseId() {
-return responseId;
-  }
-
-  @JsonProperty("responseId")
-  public void setResponseId(int responseId) {
-this.responseId = responseId;
-  }
-
-  public long getTimestamp() {
-return timestamp;
-  }
-
-  public void setTimestamp(long timestamp) {
-this.timestamp = timestamp;
-  }
-
-  public String getLabel() {
-return label;
-  }
-
-  public void setLabel(String label) {
-this.label = label;
-  }
-
-  public String getTags() {
-return tags;
-  }
-
-  public void setTags(String tags) {
-this.tags = tags;
-  }
-
-  public HostInfo getHardwareProfile() {
-return hardwareProfile;
-  }
-
-  public void setHardwareProfile(HostInfo hardwareProfile) {
-this.hardwareProfile = hardwareProfile;
-  }
-
-  public String getPublicHostname() {
-return publicHostname;
-  }
-
-  public void setPublicHostname(String name) {
-this.publicHostname = name;
-  }
-
-  public AgentEnv getAgentEnv() {
-return agentEnv;
-  }
-
-  public void setAgentEnv(AgentEnv env) {
-this.agentEnv = env;
-  }
-
-  public String getAgentVersion() {
-return agentVersion;
-  }
-
-  public void setAgentVersion(String agentVersion) {
-this.agentVersion = agentVersion;
-  }
-
-  public int getCurrentPingPort() {
-return currentPingPort;
-  }
-
-  public void setCurrentPingPort(int currentPingPort) {
-this.currentPingPort = currentPingPort;
-  }
-
-  public State getActualState() {
-return actualState;
-  }
-
-  public void setActualState(State actualState) {
-this.actualState = actualState;
-  }
-
-  public State getExpectedState() {
-return expectedState;
-  }
-
-  public void setExpectedState(State expectedState) {
-this.expectedState = expectedState;
-  }
-
-  /** @return the allocated ports, or null if none are present */
-  @JsonProperty("allocatedPorts")
-  public Map getAllocatedPorts() {
-return allocatedPorts;
-  }
-
-  /** @param ports allocated ports */
-  @JsonProperty("allocatedPorts")
-  public void setAllocatedPorts(Map ports) {
-this.allocatedPorts = ports;
-  }
-
-  /** @return the log folders, or null if none are present */
-  

[12/50] [abbrv] hadoop git commit: YARN-5735. Make the service REST API use the app timeout feature YARN-4205. Contributed by Jian He

[jianhe]

YARN-5735. Make the service REST API use the app timeout feature YARN-4205. 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7cca2928
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7cca2928
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7cca2928

Branch: refs/heads/yarn-native-services
Commit: 7cca29280d192433af5447385bb8ce3072c5d677
Parents: 6d7d4f7
Author: Gour Saha 
Authored: Fri Oct 14 17:40:51 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../api/impl/ApplicationApiService.java | 10 --
 .../org/apache/slider/client/SliderClient.java  | 33 ++--
 .../AbstractClusterBuildingActionArgs.java  |  5 +++
 .../slider/common/params/ActionThawArgs.java|  6 
 .../apache/slider/common/params/Arguments.java  |  1 +
 5 files changed, 36 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7cca2928/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 21cf113..73df4a1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -347,7 +347,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 if (queueName != null && queueName.trim().length() > 0) {
   createArgs.queue = queueName.trim();
 }
-
+createArgs.lifetime = application.getLifetime();
 return invokeSliderClientRunnable(new 
SliderClientContextRunnable() {
   @Override
   public String run(SliderClient sliderClient) throws YarnException,
@@ -1246,13 +1246,17 @@ public class ApplicationApiService implements 
ApplicationApi {
 });
   }
 
-  private Response startSliderApplication(final String appName)
+  private Response startSliderApplication(final String appName, Application 
app)
   throws IOException, YarnException, InterruptedException {
 return invokeSliderClientRunnable(new 
SliderClientContextRunnable() {
   @Override
   public Response run(SliderClient sliderClient) throws YarnException,
   IOException, InterruptedException {
 ActionThawArgs thawArgs = new ActionThawArgs();
+if (app.getLifetime() == null) {
+  app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+}
+thawArgs.lifetime = app.getLifetime();
 int returnCode = sliderClient.actionThaw(appName, thawArgs);
 if (returnCode == 0) {
   logger.info("Successfully started application {}", appName);
@@ -1344,7 +1348,7 @@ public class ApplicationApiService implements 
ApplicationApi {
   try {
 int livenessCheck = getSliderList(appName);
 if (livenessCheck != 0) {
-  return startSliderApplication(appName);
+  return startSliderApplication(appName, updateAppData);
 } else {
   logger.info("Application {} is already running", appName);
   ApplicationStatus applicationStatus = new ApplicationStatus();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7cca2928/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fe4f1d2..2840c4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.util.Shell;
 

[10/50] [abbrv] hadoop git commit: YARN-5689. Update native services REST API to use agentless docker provider. Contributed by Billie Rinaldi & Gour Saha

[jianhe]

YARN-5689. Update native services REST API to use agentless docker provider. 
Contributed by Billie Rinaldi & Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2034e9b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2034e9b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2034e9b5

Branch: refs/heads/yarn-native-services
Commit: 2034e9b52334f0aa6b8bf277b1ac38d9977aa17b
Parents: b2420ab
Author: Jian He 
Authored: Thu Oct 13 11:34:58 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../api/impl/ApplicationApiService.java | 251 ++-
 .../yarn/services/utils/RestApiConstants.java   |   3 -
 .../api/impl/TestApplicationApiService.java |   6 +-
 3 files changed, 79 insertions(+), 181 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2034e9b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 9645696..0a62629 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -50,7 +50,6 @@ import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.Response.Status;
 
-import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -68,10 +67,12 @@ import 
org.apache.hadoop.yarn.services.resource.Configuration;
 import org.apache.hadoop.yarn.services.resource.Container;
 import org.apache.hadoop.yarn.services.resource.ContainerState;
 import org.apache.hadoop.yarn.services.resource.Resource;
+import org.apache.slider.api.OptionKeys;
 import org.apache.slider.api.ResourceKeys;
 import org.apache.slider.api.StateValues;
 import org.apache.slider.client.SliderClient;
 import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.common.SliderKeys;
 import org.apache.slider.common.params.ActionCreateArgs;
 import org.apache.slider.common.params.ActionFlexArgs;
 import org.apache.slider.common.params.ActionFreezeArgs;
@@ -88,12 +89,11 @@ import org.apache.slider.core.exceptions.NotFoundException;
 import org.apache.slider.core.exceptions.SliderException;
 import org.apache.slider.core.exceptions.UnknownApplicationInstanceException;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.providers.docker.DockerKeys;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonNull;
 import com.google.gson.JsonObject;
@@ -211,7 +211,8 @@ public class ApplicationApiService implements 
ApplicationApi {
   application.setConfiguration(new Configuration());
 }
 addPropertyToConfiguration(application.getConfiguration(),
-PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+SliderKeys.COMPONENT_TYPE_KEY,
+SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
   }
   // resource
   validateApplicationResource(application.getResource(), null, application
@@ -249,7 +250,8 @@ public class ApplicationApiService implements 
ApplicationApi {
 comp.setConfiguration(new Configuration());
   }
   addPropertyToConfiguration(comp.getConfiguration(),
-  PROPERTY_COMPONENT_TYPE, COMPONENT_TYPE_EXTERNAL);
+  SliderKeys.COMPONENT_TYPE_KEY,
+  SliderKeys.COMPONENT_TYPE_EXTERNAL_APP);
   compNameArtifactIdMap.put(comp.getName(), 
comp.getArtifact().getId());
   comp.setName(comp.getArtifact().getId());
 }
@@ -339,9 +341,9 @@ public class ApplicationApiService implements 
ApplicationApi {
 final ActionCreateArgs createArgs = new ActionCreateArgs();
 addAppConfOptions(createArgs, application, compNameArtifactIdMap);
 

[22/50] [abbrv] hadoop git commit: YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron

[jianhe]

YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ba1de8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ba1de8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ba1de8f

Branch: refs/heads/yarn-native-services
Commit: 8ba1de8fffbae78a64feef5deb4384d2df404c27
Parents: 144b9f4
Author: Jian He 
Authored: Sun Jun 12 11:32:03 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 hadoop-project/pom.xml  |8 +
 .../dev-support/findbugs-exclude.xml|   15 +
 .../hadoop-yarn/hadoop-yarn-registry/pom.xml|5 +
 .../registry/client/api/DNSOperations.java  |   60 +
 .../client/api/DNSOperationsFactory.java|   78 +
 .../registry/client/api/RegistryConstants.java  |  111 +-
 .../registry/client/impl/zk/CuratorService.java |  266 ++-
 .../registry/client/impl/zk/ListenerHandle.java |   25 +
 .../registry/client/impl/zk/PathListener.java   |   30 +
 .../types/yarn/YarnRegistryAttributes.java  |   16 +-
 .../dns/ApplicationServiceRecordProcessor.java  |  353 
 .../server/dns/BaseServiceRecordProcessor.java  |  469 ++
 .../dns/ContainerServiceRecordProcessor.java|  278 
 .../server/dns/RecordCreatorFactory.java|  275 
 .../hadoop/registry/server/dns/RegistryDNS.java | 1534 ++
 .../registry/server/dns/RegistryDNSServer.java  |  290 
 .../registry/server/dns/SecureableZone.java |  151 ++
 .../server/dns/ServiceRecordProcessor.java  |   53 +
 .../registry/server/dns/ZoneSelector.java   |   33 +
 .../registry/server/dns/package-info.java   |   26 +
 .../registry/server/dns/TestRegistryDNS.java|  561 +++
 .../server/dns/TestSecureRegistryDNS.java   |   44 +
 .../test/resources/0.17.172.in-addr.arpa.zone   |   36 +
 .../src/test/resources/test.private |   32 +
 24 files changed, 4661 insertions(+), 88 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ba1de8f/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index a9c3299..41c19a1 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -84,6 +84,7 @@
 3.4.6
 2.7.1
 3.0.0
+2.1.7
 
 6.0.48
 4.0
@@ -1172,6 +1173,13 @@
   
 
   
+
+  
+dnsjava
+dnsjava
+${dnsjava.version}
+  
+
   
 org.skyscreamer
 jsonassert

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ba1de8f/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml 
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c090749..9600a93 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -579,4 +579,19 @@
 
   
 
+  
+
+
+
+  
+  
+
+
+
+  
+  
+
+
+
+  
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ba1de8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 25bfa2e..5eeea0c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -80,6 +80,11 @@
   test
 
 
+
+  dnsjava
+  dnsjava
+
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ba1de8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
new file mode 100644
index 000..3abfb6c
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/api/DNSOperations.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF 

[27/50] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
deleted file mode 100644
index 78bb8c1..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent.application.metadata;
-
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jackson.annotate.JsonProperty;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Component defined in master package metainfo.json
- */
-public class Component extends AbstractComponent {
-
-  String category = CATEGORY_MASTER;
-  String publishConfig = Boolean.FALSE.toString();
-  String minInstanceCount = "0";
-  String maxInstanceCount;
-  String autoStartOnFailure = Boolean.FALSE.toString();
-  String appExports;
-  String compExports;
-  String type = TYPE_STANDARD;
-  List componentExports = new ArrayList<>();
-  List dockerContainers = new ArrayList<>();
-  List configFiles = new ArrayList<>();
-
-  public Component() {
-  }
-
-  public String getType() {
-return type;
-  }
-
-  public void setType(String type) {
-this.type = type;
-  }
-
-  public String getCategory() {
-return category;
-  }
-
-  public void setCategory(String category) {
-this.category = category;
-  }
-
-  public String getPublishConfig() {
-return publishConfig;
-  }
-
-  public void setPublishConfig(String publishConfig) {
-this.publishConfig = publishConfig;
-  }
-
-  public String getAutoStartOnFailure() {
-return autoStartOnFailure;
-  }
-
-  public void setAutoStartOnFailure(String autoStartOnFailure) {
-this.autoStartOnFailure = autoStartOnFailure;
-  }
-
-  public String getAppExports() {
-return appExports;
-  }
-
-  public void setAppExports(String appExports) {
-this.appExports = appExports;
-  }
-
-  public String getCompExports() {
-return compExports;
-  }
-
-  public void setCompExports(String compExports) {
-this.compExports = compExports;
-  }
-
-  public String getMinInstanceCount() {
-return minInstanceCount;
-  }
-  
-  @JsonProperty("dockerContainers")
-  public List getDockerContainers() {
- return this.dockerContainers;
-  }
-  
-  public Boolean getAutoStartOnFailureBoolean() {
-if (SliderUtils.isUnset(getAutoStartOnFailure())) {
-  return Boolean.FALSE;
-}
-
-return Boolean.parseBoolean(getAutoStartOnFailure());
-  }
-
-  public int getMinInstanceCountInt() throws BadConfigException {
-if (SliderUtils.isUnset(minInstanceCount)) {
-  return 0;
-}
-
-try {
-  return Integer.parseInt(minInstanceCount);
-} catch (NumberFormatException nfe) {
-  throw new BadConfigException(nfe, "Invalid value for minInstanceCount 
for %s", name);
-}
-  }
-
-  public int getMaxInstanceCountInt() throws BadConfigException {
-if (SliderUtils.isUnset(maxInstanceCount)) {
-  return Integer.MAX_VALUE;
-}
-
-try {
-  return Integer.parseInt(maxInstanceCount);
-} catch (NumberFormatException nfe) {
-  throw new BadConfigException(nfe, "Invalid value for maxInstanceCount 
for %s", name);
-}
-  }
-
-  public void setMinInstanceCount(String minInstanceCount) {
-this.minInstanceCount = minInstanceCount;
-  }
-
-  public String getMaxInstanceCount() {
-return 

[44/50] [abbrv] hadoop git commit: YARN-5968. Fix slider core module javadocs. Contributed by Billie Rinaldi

[jianhe]

YARN-5968. Fix slider core module javadocs. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4064cec5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4064cec5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4064cec5

Branch: refs/heads/yarn-native-services
Commit: 4064cec566219e2f7f787545d715198c78a8eb0d
Parents: e2f380f
Author: Jian He 
Authored: Mon Dec 19 13:06:56 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 hadoop-yarn-project/hadoop-yarn/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4064cec5/hadoop-yarn-project/hadoop-yarn/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/pom.xml
index c43588a..4504140 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -75,7 +75,7 @@
 org.apache.maven.plugins
 maven-javadoc-plugin
 
-  
org.apache.hadoop.yarn.proto
+  
org.apache.hadoop.yarn.proto,org.apache.slider
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[24/50] [abbrv] hadoop git commit: YARN-5808. Add gc log options to the yarn daemon script when starting services-api. Contributed by Billie Rinaldi

[jianhe]

YARN-5808. Add gc log options to the yarn daemon script when starting 
services-api. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8c50e10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8c50e10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8c50e10

Branch: refs/heads/yarn-native-services
Commit: c8c50e10fdd7ef8f35184339ed7a6fb7c920daf3
Parents: 882deb1
Author: Gour Saha 
Authored: Thu Nov 10 11:35:02 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 hadoop-yarn-project/hadoop-yarn/bin/yarn | 16 
 hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh | 12 
 2 files changed, 20 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8c50e10/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 63f1459..be4a326 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -47,7 +47,7 @@ function hadoop_usage
   hadoop_add_subcommand "resourcemanager" "run the ResourceManager"
   hadoop_add_subcommand "rmadmin" "admin tools"
   hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
-  hadoop_add_subcommand "services-api" "run slider services api"
+  hadoop_add_subcommand "servicesapi" "run slider services api"
   hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager 
daemon"
   hadoop_add_subcommand "slider" "run a slider app"
   hadoop_add_subcommand "timelinereader" "run the timeline reader server"
@@ -144,20 +144,20 @@ function yarncmd_case
 scmadmin)
   HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
 ;;
-services-api)
+servicesapi)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/services-api"'/*'
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.services.webapp.ApplicationApiWebApp'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 sharedcachemanager)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -166,15 +166,15 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
 slider)
   hadoop_add_classpath 
"${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider"'/*'
   HADOOP_CLASSNAME='org.apache.slider.Slider'
-  hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
-  HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS} \
--Dslider.libdir=${HADOOP_YARN_HOME}/${YARN_DIR},\
+  local sld="${HADOOP_YARN_HOME}/${YARN_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR},\
 ${HADOOP_YARN_HOME}/${YARN_LIB_JARS_DIR}/slider,\
 ${HADOOP_HDFS_HOME}/${HDFS_DIR},\
 ${HADOOP_HDFS_HOME}/${HDFS_LIB_JARS_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_DIR},\
 ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
+  hadoop_translate_cygwin_path sld
+  hadoop_add_param HADOOP_OPTS slider.libdir "-Dslider.libdir=${sld}"
 ;;
 timelinereader)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8c50e10/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
--
diff --git a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh 
b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
index d003adb..3828897 100644
--- a/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
+++ b/hadoop-yarn-project/hadoop-yarn/conf/yarn-env.sh
@@ -136,3 +136,15 @@
 # See ResourceManager for some examples
 #
 #export YARN_SHAREDCACHEMANAGER_OPTS=
+
+###
+# Services API specific parameters
+###
+# Specify the JVM options to be used when starting the services API.
+#
+# These options will be appended to the options specified as HADOOP_OPTS
+# and therefore may override any similar flags set in HADOOP_OPTS
+#
+# See ResourceManager for some examples
+#
+#export YARN_SERVICESAPI_OPTS="-verbose:gc -XX:+PrintGCDetails 
-XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps 

[47/50] [abbrv] hadoop git commit: YARN-6010. Fix findbugs, site warnings in yarn-services-api module. Contributed by Jian He

[jianhe]

YARN-6010. Fix findbugs, site warnings in yarn-services-api module. Contributed 
by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9e13bd35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9e13bd35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9e13bd35

Branch: refs/heads/yarn-native-services
Commit: 9e13bd354811593de73e2f56ce99a3c0d43b4965
Parents: 4064cec
Author: Billie Rinaldi 
Authored: Mon Dec 19 16:15:58 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../dev-support/findbugs-exclude.xml   |  4 +++-
 .../hadoop-yarn-services-api/pom.xml   | 13 -
 2 files changed, 3 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e13bd35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
index b89146a..2843338 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -16,5 +16,7 @@
limitations under the License.
 -->
 
-
+
+
+
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9e13bd35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index c198f28..e467f7f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -108,19 +108,6 @@
 
   
 
-  
-org.apache.maven.plugins
-maven-site-plugin
-${maven-site-plugin.version}
-
-
-  
-org.apache.maven.doxia
-doxia-module-markdown
-${maven-doxia-module-markdown.version}
-  
-
-  
 
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[13/50] [abbrv] hadoop git commit: YARN-5775. Convert enums in swagger definition to uppercase. Contributed by Gour Saha

[jianhe]

YARN-5775. Convert enums in swagger definition to uppercase. Contributed by 
Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e5a4deb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e5a4deb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e5a4deb

Branch: refs/heads/yarn-native-services
Commit: 5e5a4deb1e37b7942ee37151f4f992383077b6a4
Parents: c232d79
Author: Billie Rinaldi 
Authored: Tue Oct 25 11:25:51 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml | 38 ++--
 1 file changed, 19 insertions(+), 19 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5a4deb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
index 6169fcd..7eb3196 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Simplified-V1-API-Layer-For-Services.yaml
@@ -241,10 +241,10 @@ definitions:
 type: string
 description: Artifact type, like docker, tarball, etc. (optional).
 enum:
-  - docker
-  - tarball
-  - application
-default: docker
+  - DOCKER
+  - TARBALL
+  - APPLICATION
+default: DOCKER
   uri:
 type: string
 description: Artifact location to support multiple artifact stores 
(optional).
@@ -303,7 +303,7 @@ definitions:
 type: string
 description: E.g. HTTP (YARN will perform a simple REST call at a 
regular interval and expect a 204 No content).
 enum:
-  - http
+  - HTTP
   uri:
 type: string
 description: Fully qualified REST uri endpoint.
@@ -335,13 +335,13 @@ definitions:
 type: string
 description: Config file in the standard format like xml, properties, 
json, yaml, template.
 enum:
-  - xml
-  - properties
-  - json
-  - yaml
-  - template
-  - env
-  - hadoop_xml
+  - XML
+  - PROPERTIES
+  - JSON
+  - YAML
+  - TEMPLATE
+  - ENV
+  - HADOOP_XML
   dest_file:
 type: string
 description: The absolute path that this configuration file should be 
mounted as, in the application container.
@@ -386,11 +386,11 @@ definitions:
 type: string
 description: enum of the state of the application
 enum:
-  - accepted
-  - started
-  - ready
-  - stopped
-  - failed
+  - ACCEPTED
+  - STARTED
+  - READY
+  - STOPPED
+  - FAILED
   ContainerState:
 description: The current state of the container of an application.
 properties:
@@ -398,8 +398,8 @@ definitions:
 type: string
 description: enum of the state of the container
 enum:
-  - init
-  - ready
+  - INIT
+  - READY
   ApplicationStatus:
 description: The current status of a submitted application, returned as a 
response to the GET API.
 properties:


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[49/50] [abbrv] hadoop git commit: Updated pom to point to 3.0.0-alpha3-SNAPSHOT

[jianhe]

Updated pom to point to 3.0.0-alpha3-SNAPSHOT


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/732c5714
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/732c5714
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/732c5714

Branch: refs/heads/yarn-native-services
Commit: 732c5714e2b08561c2229351d5a82560c29678f5
Parents: 585fc04
Author: Jian He 
Authored: Tue Feb 7 14:38:32 2017 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml| 4 ++--
 .../hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml   | 2 +-
 .../hadoop-yarn-applications/hadoop-yarn-slider/pom.xml  | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/732c5714/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
index e467f7f..4e88aef 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -19,12 +19,12 @@
   
 org.apache.hadoop
 hadoop-yarn-applications
-3.0.0-alpha2-SNAPSHOT
+3.0.0-alpha3-SNAPSHOT
   
   org.apache.hadoop
   hadoop-yarn-services-api
   Apache Hadoop YARN Services API
-  3.0.0-alpha2-SNAPSHOT
+  3.0.0-alpha3-SNAPSHOT
   jar
   Hadoop YARN REST APIs for services
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/732c5714/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
index 81e607b..084a10b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/pom.xml
@@ -19,7 +19,7 @@
   
 org.apache.hadoop
 hadoop-yarn-slider
-3.0.0-alpha2-SNAPSHOT
+3.0.0-alpha3-SNAPSHOT
   
   org.apache.hadoop
   hadoop-yarn-slider-core

http://git-wip-us.apache.org/repos/asf/hadoop/blob/732c5714/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
index 780f68b..cc42b18 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/pom.xml
@@ -19,7 +19,7 @@
 
 hadoop-yarn-applications
 org.apache.hadoop
-3.0.0-alpha2-SNAPSHOT
+3.0.0-alpha3-SNAPSHOT
 
 4.0.0
 org.apache.hadoop


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[20/50] [abbrv] hadoop git commit: YARN-5218. Initial core change for DNS for YARN. Contributed by Jonathan Maron

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ba1de8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
new file mode 100644
index 000..37f0d23
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -0,0 +1,561 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.server.dns;
+
+import org.apache.commons.net.util.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryConstants;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.ARecord;
+import org.xbill.DNS.CNAMERecord;
+import org.xbill.DNS.DClass;
+import org.xbill.DNS.DNSKEYRecord;
+import org.xbill.DNS.DNSSEC;
+import org.xbill.DNS.Flags;
+import org.xbill.DNS.Message;
+import org.xbill.DNS.Name;
+import org.xbill.DNS.OPTRecord;
+import org.xbill.DNS.PTRRecord;
+import org.xbill.DNS.RRSIGRecord;
+import org.xbill.DNS.RRset;
+import org.xbill.DNS.Rcode;
+import org.xbill.DNS.Record;
+import org.xbill.DNS.SRVRecord;
+import org.xbill.DNS.Section;
+import org.xbill.DNS.Type;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.net.Inet6Address;
+import java.net.InetAddress;
+import java.security.KeyFactory;
+import java.security.PrivateKey;
+import java.security.spec.RSAPrivateKeySpec;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.concurrent.TimeUnit;
+
+import static 
org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_MASK;
+import static 
org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_ZONE_SUBNET;
+
+/**
+ *
+ */
+public class TestRegistryDNS extends Assert {
+
+  private RegistryDNS registryDNS;
+  private RegistryUtils.ServiceRecordMarshal marshal;
+
+  private static final String APPLICATION_RECORD = "{\n"
+  + "  \"type\" : \"JSONServiceRecord\",\n"
+  + "  \"description\" : \"Slider Application Master\",\n"
+  + "  \"external\" : [ {\n"
+  + "\"api\" : \"classpath:org.apache.slider.appmaster.ipc\",\n"
+  + "\"addressType\" : \"host/port\",\n"
+  + "\"protocolType\" : \"hadoop/IPC\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"host\" : \"192.168.1.5\",\n"
+  + "  \"port\" : \"1026\"\n"
+  + "} ]\n"
+  + "  }, {\n"
+  + "\"api\" : \"http://\",\n;
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"webui\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"http://192.168.1.5:1027\"\n;
+  + "} ]\n"
+  + "  }, {\n"
+  + "\"api\" : \"classpath:org.apache.slider.management\",\n"
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"REST\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"http://192.168.1.5:1027/ws/v1/slider/mgmt\"\n;
+  + "} ]\n"
+  + "  } ],\n"
+  + "  \"internal\" : [ {\n"
+  + "\"api\" : \"classpath:org.apache.slider.agents.secure\",\n"
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"REST\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"https://192.168.1.5:47700/ws/v1/slider/agents\"\n;
+  + "} ]\n"
+  + "  }, {\n"
+  + "\"api\" : \"classpath:org.apache.slider.agents.oneway\",\n"
+  + "\"addressType\" : \"uri\",\n"
+  + "\"protocolType\" : \"REST\",\n"
+  + "\"addresses\" : [ {\n"
+  + "  \"uri\" : \"https://192.168.1.5:35531/ws/v1/slider/agents\"\n;
+  + "

[17/50] [abbrv] hadoop git commit: YARN-5740. Add a new field in Slider status output - lifetime (remaining). Contributed by Jian He

[jianhe]

YARN-5740. Add a new field in Slider status output - lifetime (remaining). 
Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9bb1acd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9bb1acd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9bb1acd

Branch: refs/heads/yarn-native-services
Commit: b9bb1acdc06dd84b14cdc68f53967896bce6132f
Parents: 7a06f94
Author: Gour Saha 
Authored: Fri Dec 16 12:03:51 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../api/impl/ApplicationApiService.java | 82 
 .../org/apache/slider/client/SliderClient.java  | 62 +++
 .../slider/common/params/ActionStatusArgs.java  |  4 +
 3 files changed, 103 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9bb1acd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index c4f5d43..b11da2c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -52,6 +52,8 @@ import org.apache.commons.lang.SerializationUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.services.api.ApplicationApi;
@@ -771,7 +773,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 
 // Get all applications in a specific state - lighter projection. For full
 // detail, call getApplication on a specific app.
-Set applications;
+Set applications;
 try {
   if (StringUtils.isNotEmpty(state)) {
 ApplicationStatus appStatus = new ApplicationStatus();
@@ -793,13 +795,12 @@ public class ApplicationApiService implements 
ApplicationApi {
 Set apps = new HashSet();
 if (applications.size() > 0) {
   try {
-for (String app : applications) {
+for (ApplicationReport app : applications) {
   Application application = new Application();
-  // TODO: Need to get lifetime, launch-time and privileged container
-  // status from YARN
-  application.setLifetime(null);
-  application.setLaunchTime(new Date());
-  application.setName(app);
+  application.setLifetime(app.getApplicationTimeouts().get(
+  ApplicationTimeoutType.LIFETIME).getRemainingTime());
+  application.setLaunchTime(new Date(app.getStartTime()));
+  application.setName(app.getName());
   // Containers not required, setting to null to avoid empty list
   application.setContainers(null);
   apps.add(application);
@@ -930,9 +931,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 app.setLaunchTime(appStatus.get("createTime") == null ? null
 : new Date(appStatus.get("createTime").getAsLong()));
 
-// lifetime - set it to unlimited for now
-// TODO: Once YARN-3813 and YARN-4205 are available - get it from YARN
-app.setLifetime(DEFAULT_UNLIMITED_LIFETIME);
+app.setLifetime(queryLifetime(appName));
 
 // Quicklinks
 Map appQuicklinks = new HashMap<>();
@@ -1062,6 +1061,24 @@ public class ApplicationApiService implements 
ApplicationApi {
 return object.get(key) == null ? null : object.get(key).getAsJsonObject();
   }
 
+  private long queryLifetime(String appName) {
+try {
+  return invokeSliderClientRunnable(
+  new SliderClientContextRunnable() {
+@Override
+public Long run(SliderClient sliderClient)
+throws YarnException, IOException, InterruptedException {
+  ApplicationReport report = sliderClient.findInstance(appName);
+   

[40/50] [abbrv] hadoop git commit: YARN-6115. Few additional paths in Slider client still uses get all Applications without tags/states filter. Contributed by Gour Saha

[jianhe]

YARN-6115. Few additional paths in Slider client still uses get all 
Applications without tags/states filter. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37cb3c54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37cb3c54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37cb3c54

Branch: refs/heads/yarn-native-services
Commit: 37cb3c54c5649e205f0a68f1b8f04e0b1ad0fd4a
Parents: fa73e1d
Author: Billie Rinaldi 
Authored: Wed Jan 25 14:20:58 2017 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/client/SliderClient.java  | 43 +++--
 .../slider/core/registry/YarnAppListClient.java | 51 +---
 2 files changed, 73 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/37cb3c54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 1a959d6..032eb6e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -142,7 +142,6 @@ import org.apache.slider.core.launch.ClasspathConstructor;
 import org.apache.slider.core.launch.CredentialUtils;
 import org.apache.slider.core.launch.JavaCommandLineBuilder;
 import org.apache.slider.core.launch.LaunchedApplication;
-import org.apache.slider.core.launch.RunningApplication;
 import org.apache.slider.core.launch.SerializedApplicationReport;
 import org.apache.slider.core.main.RunService;
 import org.apache.slider.core.persist.AppDefinitionPersister;
@@ -1535,7 +1534,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 
   public void updateLifetime(String appName, long lifetime)
   throws YarnException, IOException {
-ApplicationReport report = findInstance(appName);
+EnumSet appStates = EnumSet.range(
+YarnApplicationState.NEW, YarnApplicationState.RUNNING);
+ApplicationReport report = findInstance(appName, appStates);
 if (report == null) {
   throw new YarnException("Application not found for " + appName);
 }
@@ -2672,15 +2673,17 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   }
 
   /**
-   * List Slider instances belonging to a specific user. This will include
-   * failed and killed instances; there may be duplicates
+   * List Slider instances belonging to a specific user with a specific app
+   * name and within a set of app states.
* @param user user: "" means all users, null means "default"
+   * @param appName name of the application set as a tag
+   * @param appStates a set of states the applications should be in
* @return a possibly empty list of Slider AMs
*/
-
-  public List listSliderInstances(String user)
-throws YarnException, IOException {
-return yarnAppListClient.listInstances(user);
+  public List listSliderInstances(String user,
+  String appName, EnumSet appStates)
+  throws YarnException, IOException {
+return yarnAppListClient.listInstances(user, appName, appStates);
   }
 
   /**
@@ -2806,7 +2809,9 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 }
 
 // and those the RM knows about
-List instances = listSliderInstances(null);
+EnumSet appStates = EnumSet.range(min, max);
+List instances = listSliderInstances(null, clustername,
+appStates);
 sortApplicationsByMostRecent(instances);
 Map reportMap =
 buildApplicationReportMap(instances, min, max);
@@ -3053,7 +3058,7 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   }
 
   /**
-   * Find an instance of an application belonging to the current user
+   * Find an instance of an application belonging to the current user.
* @param appname application name
* @return the app report or null if none is found
* @throws YarnException YARN issues
@@ -3061,14 +3066,22 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
*/
 

[35/50] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc72ab2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
deleted file mode 100644
index 081b7fa..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+++ /dev/null
@@ -1,2293 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: SliderClusterProtocol.proto
-
-package org.apache.slider.api.proto;
-
-public final class SliderClusterAPI {
-  private SliderClusterAPI() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  /**
-   * Protobuf service {@code org.apache.slider.api.SliderClusterProtocolPB}
-   *
-   * 
-   **
-   * Protocol used from between Slider Client and AM
-   * 
-   */
-  public static abstract class SliderClusterProtocolPB
-  implements com.google.protobuf.Service {
-protected SliderClusterProtocolPB() {}
-
-public interface Interface {
-  /**
-   * rpc stopCluster(.org.apache.slider.api.StopClusterRequestProto) 
returns (.org.apache.slider.api.StopClusterResponseProto);
-   */
-  public abstract void stopCluster(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.StopClusterRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
upgradeContainers(.org.apache.slider.api.UpgradeContainersRequestProto) returns 
(.org.apache.slider.api.UpgradeContainersResponseProto);
-   *
-   * 
-   **
-   * Upgrade containers 
-   * 
-   */
-  public abstract void upgradeContainers(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.UpgradeContainersRequestProto 
request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc flexCluster(.org.apache.slider.api.FlexClusterRequestProto) 
returns (.org.apache.slider.api.FlexClusterResponseProto);
-   *
-   * 
-   **
-   * Flex the cluster. 
-   * 
-   */
-  public abstract void flexCluster(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.FlexClusterRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
getJSONClusterStatus(.org.apache.slider.api.GetJSONClusterStatusRequestProto) 
returns (.org.apache.slider.api.GetJSONClusterStatusResponseProto);
-   *
-   * 
-   **
-   * Get the current cluster status
-   * 
-   */
-  public abstract void getJSONClusterStatus(
-  com.google.protobuf.RpcController controller,
-  
org.apache.slider.api.proto.Messages.GetJSONClusterStatusRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
getInstanceDefinition(.org.apache.slider.api.GetInstanceDefinitionRequestProto) 
returns (.org.apache.slider.api.GetInstanceDefinitionResponseProto);
-   *
-   * 
-   **
-   * Get the instance definition
-   * 
-   */
-  public abstract void getInstanceDefinition(
-  com.google.protobuf.RpcController controller,
-  
org.apache.slider.api.proto.Messages.GetInstanceDefinitionRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 
listNodeUUIDsByRole(.org.apache.slider.api.ListNodeUUIDsByRoleRequestProto) 
returns (.org.apache.slider.api.ListNodeUUIDsByRoleResponseProto);
-   *
-   * 
-   **
-   * List all running nodes in a role
-   * 
-   */
-  public abstract void listNodeUUIDsByRole(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.ListNodeUUIDsByRoleRequestProto 
request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc getNode(.org.apache.slider.api.GetNodeRequestProto) returns 
(.org.apache.slider.api.GetNodeResponseProto);
-   *
-   * 
-   **
-   * Get the details on a node
-   * 
-   */
-  public abstract void getNode(
-  com.google.protobuf.RpcController controller,
-  org.apache.slider.api.proto.Messages.GetNodeRequestProto request,
-  
com.google.protobuf.RpcCallback
 done);
-
-  /**
-   * rpc 

[26/50] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index a0fe310..bd4d2bf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -23,7 +23,6 @@ import org.apache.slider.server.appmaster.actions.QueueAccess;
 import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
 import org.apache.slider.server.appmaster.state.RoleStatus;
 import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
 import 
org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
 import org.apache.slider.server.services.security.CertificateManager;
 import org.slf4j.Logger;
@@ -97,11 +96,6 @@ public class WebAppApiImpl implements WebAppApi {
   }
 
   @Override
-  public AgentRestOperations getAgentRestOperations() {
-return provider.getAgentRestOperations();
-  }
-
-  @Override
   public RegistryOperations getRegistryOperations() {
 return registryOperations;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
index 03bf703..aed87d8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AMWebServices.java
@@ -19,7 +19,6 @@ package org.apache.slider.server.appmaster.web.rest;
 import com.google.inject.Inject;
 import com.google.inject.Singleton;
 import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.agent.AgentResource;
 import 
org.apache.slider.server.appmaster.web.rest.application.ApplicationResource;
 import 
org.apache.slider.server.appmaster.web.rest.management.ManagementResource;
 import org.apache.slider.server.appmaster.web.rest.publisher.PublisherResource;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
deleted file mode 100644
index 17cd8f2..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentCommandType.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless 

[23/50] [abbrv] hadoop git commit: YARN-5828. Native services client errors out when config formats are uppercase. Contributed by Billie Rinaldi

[jianhe]

YARN-5828. Native services client errors out when config formats are uppercase. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/882deb14
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/882deb14
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/882deb14

Branch: refs/heads/yarn-native-services
Commit: 882deb143bbc030b0c75abf842deea59ac96a806
Parents: 5dbff3c
Author: Gour Saha 
Authored: Thu Nov 3 18:15:44 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/core/registry/docstore/ConfigFormat.java   | 4 +++-
 .../src/main/java/org/apache/slider/providers/ProviderUtils.java | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/882deb14/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index ddab606..723b975 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -18,6 +18,8 @@
 
 package org.apache.slider.core.registry.docstore;
 
+import java.util.Locale;
+
 public enum ConfigFormat {
 
   JSON("json"),
@@ -51,7 +53,7 @@ public enum ConfigFormat {
*/
   public static ConfigFormat resolve(String type) {
 for (ConfigFormat format: values()) {
-  if (format.getSuffix().equals(type)) {
+  if (format.getSuffix().equals(type.toLowerCase(Locale.ENGLISH))) {
 return format;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/882deb14/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index c5e6782..39986c1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -611,7 +611,7 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
   }
   ConfigFormat configFormat = ConfigFormat.resolve(configFileType);
   if (configFormat == null) {
-throw new BadConfigException("Config format " + configFormat +
+throw new BadConfigException("Config format " + configFileType +
 " doesn't exist");
   }
   localizeConfigFile(launcher, roleName, roleGroup, configEntry.getKey(),


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[33/50] [abbrv] hadoop git commit: YARN-5941. Slider handles "per.component" for multiple components incorrectly. Contributed by Billie Rinaldi

[jianhe]

YARN-5941. Slider handles "per.component" for multiple components incorrectly. 
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edcf09f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edcf09f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edcf09f5

Branch: refs/heads/yarn-native-services
Commit: edcf09f5de06b3d258dd5761f793263de4b095ab
Parents: 9a52d73
Author: Gour Saha 
Authored: Wed Nov 30 14:00:22 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../java/org/apache/slider/api/OptionKeys.java  | 14 ++
 .../org/apache/slider/common/SliderKeys.java|  2 --
 .../apache/slider/providers/ProviderUtils.java  | 20 
 3 files changed, 30 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcf09f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index 434b1d9..988627d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -53,8 +53,22 @@ public interface OptionKeys extends InternalKeys {
* Prefix for export options: {@value}
*/
   String EXPORT_PREFIX = "export.";
+  /**
+   * Type suffix for config file and package options: {@value}
+   */
   String TYPE_SUFFIX = ".type";
+  /**
+   * Name suffix for config file and package options: {@value}
+   */
   String NAME_SUFFIX = ".name";
+  /**
+   * Per component suffix for config file options: {@value}
+   */
+  String PER_COMPONENT = ".per.component";
+  /**
+   * Per group suffix for config file options: {@value}
+   */
+  String PER_GROUP = ".per.group";
 
   /**
* Zookeeper quorum host list: {@value}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcf09f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 3d25d33..adf40ce 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -304,8 +304,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
 
   String APP_RESOURCES = "application.resources";
   String APP_RESOURCES_DIR = "app/resources";
-  String PER_COMPONENT = "per.component";
-  String PER_GROUP = "per.group";
 
   String APP_PACKAGES_DIR = "app/packages";
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/edcf09f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index 39986c1..bc237f5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -614,8 +614,16 @@ public class ProviderUtils implements RoleKeys, 

[38/50] [abbrv] hadoop git commit: YARN-5944. Native services AM should remain up if RM is down. Contributed by Billie Rinaldi

[jianhe]

YARN-5944. Native services AM should remain up if RM is down. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9203dc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9203dc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9203dc9

Branch: refs/heads/yarn-native-services
Commit: a9203dc96d9fd4cb515f1d852cfbd922e323eb69
Parents: c916a0f
Author: Gour Saha 
Authored: Thu Dec 1 00:30:01 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/server/appmaster/SliderAppMaster.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9203dc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 34b6a7d..8c39343 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -705,6 +705,11 @@ public class SliderAppMaster extends 
AbstractSliderLaunchedService
 synchronized (appState) {
   int heartbeatInterval = HEARTBEAT_INTERVAL;
 
+  // configure AM to wait forever for RM
+  
getConfig().setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+  -1);
+  getConfig().unset(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS);
+
   // add the RM client -this brings the callbacks in
   asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, 
this);
   addService(asyncRMClient);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[02/50] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb860960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8203cf0..fdc5be1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -82,6 +82,8 @@ import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
 
+import static 
org.apache.slider.common.tools.SliderUtils.getApplicationDefinitionPath;
+
 /** This class implements  the client-side aspects of the agent deployer */
 public class AgentClientProvider extends AbstractClientProvider
 implements AgentKeys, SliderKeys {
@@ -132,13 +134,13 @@ public class AgentClientProvider extends 
AbstractClientProvider
 sliderFileSystem.verifyFileExists(appDefPath);
 
 String agentConf = instanceDefinition.getAppConfOperations().
-getGlobalOptions().getOption(AgentKeys.AGENT_CONF, "");
+getGlobalOptions().getOption(AGENT_CONF, "");
 if (StringUtils.isNotEmpty(agentConf)) {
   sliderFileSystem.verifyFileExists(new Path(agentConf));
 }
 
 String appHome = instanceDefinition.getAppConfOperations().
-getGlobalOptions().get(AgentKeys.PACKAGE_PATH);
+getGlobalOptions().get(PACKAGE_PATH);
 if (SliderUtils.isUnset(appHome)) {
   String agentImage = instanceDefinition.getInternalOperations().
   get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
@@ -173,7 +175,7 @@ public class AgentClientProvider extends 
AbstractClientProvider
 }
 
 Set names = resources.getComponentNames();
-names.remove(SliderKeys.COMPONENT_AM);
+names.remove(COMPONENT_AM);
 Map priorityMap = new HashMap();
 
 for (String name : names) {
@@ -271,7 +273,7 @@ public class AgentClientProvider extends 
AbstractClientProvider
 String agentImage = instanceDefinition.getInternalOperations().
 get(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
 if (SliderUtils.isUnset(agentImage)) {
-  Path agentPath = new Path(tempPath.getParent(), 
AgentKeys.PROVIDER_AGENT);
+  Path agentPath = new Path(tempPath.getParent(), PROVIDER_AGENT);
   log.info("Automatically uploading the agent tarball at {}", agentPath);
   fileSystem.getFileSystem().mkdirs(agentPath);
   if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
@@ -284,6 +286,12 @@ public class AgentClientProvider extends 
AbstractClientProvider
 
   @Override
   public Set getApplicationTags(SliderFileSystem fileSystem,
+  ConfTreeOperations appConf) throws SliderException {
+return getApplicationTags(fileSystem,
+getApplicationDefinitionPath(appConf));
+  }
+
+  public Set getApplicationTags(SliderFileSystem fileSystem,
 String appDef) throws SliderException {
 Set tags;
 Metainfo metaInfo = getMetainfo(fileSystem, appDef);
@@ -437,19 +445,19 @@ public class AgentClientProvider extends 
AbstractClientProvider
 if (config != null) {
   try {
 clientRoot = config.getJSONObject("global")
-.getString(AgentKeys.APP_CLIENT_ROOT);
+.getString(APP_CLIENT_ROOT);
   } catch (JSONException e) {
 log.info("Couldn't read {} from provided client config, falling " +
-"back on default", AgentKeys.APP_CLIENT_ROOT);
+"back on default", APP_CLIENT_ROOT);
   }
 }
 if (clientRoot == null && defaultConfig != null) {
   try {
 clientRoot = defaultConfig.getJSONObject("global")
-.getString(AgentKeys.APP_CLIENT_ROOT);
+.getString(APP_CLIENT_ROOT);
   } catch (JSONException e) {
 log.info("Couldn't read {} from default client config, using {}",
-AgentKeys.APP_CLIENT_ROOT, clientInstallPath);
+APP_CLIENT_ROOT, clientInstallPath);
   }
 }
 if (clientRoot == null) {
@@ -500,7 +508,7 @@ public class AgentClientProvider extends 
AbstractClientProvider
 try {
   String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + 

[36/50] [abbrv] hadoop git commit: YARN-5961. Generate native services protobuf classes during build. Contributed by Billie Rinaldi

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cc72ab2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
deleted file mode 100644
index 373d64d..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ /dev/null
@@ -1,34473 +0,0 @@
-// Generated by the protocol buffer compiler.  DO NOT EDIT!
-// source: SliderClusterMessages.proto
-
-package org.apache.slider.api.proto;
-
-public final class Messages {
-  private Messages() {}
-  public static void registerAllExtensions(
-  com.google.protobuf.ExtensionRegistry registry) {
-  }
-  public interface RoleInstanceStateOrBuilder
-  extends com.google.protobuf.MessageOrBuilder {
-
-// required string name = 1;
-/**
- * required string name = 1;
- */
-boolean hasName();
-/**
- * required string name = 1;
- */
-java.lang.String getName();
-/**
- * required string name = 1;
- */
-com.google.protobuf.ByteString
-getNameBytes();
-
-// optional string role = 2;
-/**
- * optional string role = 2;
- */
-boolean hasRole();
-/**
- * optional string role = 2;
- */
-java.lang.String getRole();
-/**
- * optional string role = 2;
- */
-com.google.protobuf.ByteString
-getRoleBytes();
-
-// required uint32 state = 4;
-/**
- * required uint32 state = 4;
- */
-boolean hasState();
-/**
- * required uint32 state = 4;
- */
-int getState();
-
-// required uint32 exitCode = 5;
-/**
- * required uint32 exitCode = 5;
- */
-boolean hasExitCode();
-/**
- * required uint32 exitCode = 5;
- */
-int getExitCode();
-
-// optional string command = 6;
-/**
- * optional string command = 6;
- */
-boolean hasCommand();
-/**
- * optional string command = 6;
- */
-java.lang.String getCommand();
-/**
- * optional string command = 6;
- */
-com.google.protobuf.ByteString
-getCommandBytes();
-
-// optional string diagnostics = 7;
-/**
- * optional string diagnostics = 7;
- */
-boolean hasDiagnostics();
-/**
- * optional string diagnostics = 7;
- */
-java.lang.String getDiagnostics();
-/**
- * optional string diagnostics = 7;
- */
-com.google.protobuf.ByteString
-getDiagnosticsBytes();
-
-// repeated string output = 8;
-/**
- * repeated string output = 8;
- */
-java.util.List
-getOutputList();
-/**
- * repeated string output = 8;
- */
-int getOutputCount();
-/**
- * repeated string output = 8;
- */
-java.lang.String getOutput(int index);
-/**
- * repeated string output = 8;
- */
-com.google.protobuf.ByteString
-getOutputBytes(int index);
-
-// repeated string environment = 9;
-/**
- * repeated string environment = 9;
- */
-java.util.List
-getEnvironmentList();
-/**
- * repeated string environment = 9;
- */
-int getEnvironmentCount();
-/**
- * repeated string environment = 9;
- */
-java.lang.String getEnvironment(int index);
-/**
- * repeated string environment = 9;
- */
-com.google.protobuf.ByteString
-getEnvironmentBytes(int index);
-
-// required uint32 roleId = 10;
-/**
- * required uint32 roleId = 10;
- */
-boolean hasRoleId();
-/**
- * required uint32 roleId = 10;
- */
-int getRoleId();
-
-// required bool released = 11;
-/**
- * required bool released = 11;
- */
-boolean hasReleased();
-/**
- * required bool released = 11;
- */
-boolean getReleased();
-
-// required int64 createTime = 12;
-/**
- * required int64 createTime = 12;
- */
-boolean hasCreateTime();
-/**
- * required int64 createTime = 12;
- */
-long getCreateTime();
-
-// required int64 startTime = 13;
-/**
- * required int64 startTime = 13;
- */
-boolean hasStartTime();
-/**
- * required int64 startTime = 13;
- */
-long getStartTime();
-
-// required string host = 14;
-/**
- * required string host = 14;
- */
-boolean hasHost();
-/**
- * required string host = 14;
- */
-java.lang.String getHost();
-/**
- * required string host = 14;
- */
-

[28/50] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
deleted file mode 100644
index 281895a..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentRoles.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import org.apache.slider.providers.ProviderRole;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class AgentRoles {
-
-  /**
-   * List of roles Agent provider does not have any roles by default. All 
roles are read from the application
-   * specification.
-   */
-  protected static final List ROLES =
-  new ArrayList();
-
-  public static List getRoles() {
-return ROLES;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
deleted file mode 100644
index 23e05a3..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.agent;
-
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import 
org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
-import 
org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import 
org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
-import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-
-import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
-

[39/50] [abbrv] hadoop git commit: YARN-5690. Integrate native services modules into maven build. Contributed by Billie Rinaldi

[jianhe]

YARN-5690. Integrate native services modules into maven build. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ca2e629
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ca2e629
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ca2e629

Branch: refs/heads/yarn-native-services
Commit: 2ca2e629d5317194f29a12db8a7d6e5033bad65d
Parents: 4163e9a
Author: Gour Saha 
Authored: Thu Oct 27 08:50:36 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../resources/assemblies/hadoop-yarn-dist.xml   | 26 
 .../assemblies/hadoop-yarn-services-api.xml | 36 +++
 .../assemblies/hadoop-yarn-slider-dist.xml  | 30 +
 hadoop-project/pom.xml  |  6 ++
 hadoop-yarn-project/hadoop-yarn/bin/yarn| 30 +
 .../hadoop-yarn-services-api/pom.xml| 44 +++--
 .../conf/slideram-log4j.properties  | 68 
 .../hadoop-yarn-slider-core/pom.xml | 38 +++
 .../org/apache/slider/client/SliderClient.java  | 29 +++--
 .../org/apache/slider/common/SliderKeys.java|  2 +-
 .../apache/slider/common/tools/SliderUtils.java | 48 ++
 .../providers/agent/AgentClientProvider.java|  3 +-
 .../slideram/SliderAMClientProvider.java| 15 +++--
 .../TestPublishedConfigurationOutputter.java| 10 ++-
 14 files changed, 343 insertions(+), 42 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ca2e629/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
index 05e1a93..391a7f5 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
@@ -86,6 +86,32 @@
   
 
 
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target
+  
/share/hadoop/${hadoop.component}/sources
+  
+*-sources.jar
+  
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/conf
+  etc/hadoop
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/target/hadoop-yarn-slider-core-${project.version}
+  
/share/hadoop/${hadoop.component}/lib/slider
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target
+  
/share/hadoop/${hadoop.component}/sources
+  
+*-sources.jar
+  
+
+
+  
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/target/hadoop-yarn-services-api-${project.version}
+  
/share/hadoop/${hadoop.component}/lib/services-api
+
+
   
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/target
   
/share/hadoop/${hadoop.component}/sources
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ca2e629/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
new file mode 100644
index 000..589f724
--- /dev/null
+++ 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-services-api.xml
@@ -0,0 +1,36 @@
+
+http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0;
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance;
+  
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0
 http://maven.apache.org/xsd/assembly-1.1.0.xsd;>
+  hadoop-yarn-services-api-dist
+  
+dir
+  
+  false
+  
+
+  false
+  
+com.fasterxml.jackson.jaxrs:jackson-jaxrs-base
+
com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider
+
com.fasterxml.jackson.module:jackson-module-jaxb-annotations
+io.swagger:swagger-annotations
+  
+
+  
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ca2e629/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
--
diff --git 
a/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml 
b/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-slider-dist.xml
new file mode 100644
index 000..5de45a9
--- /dev/null
+++ 

[29/50] [abbrv] hadoop git commit: YARN-5909. Remove agent related code in slider AM. Contributed by Jian He

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf5ffbb9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
deleted file mode 100644
index 499812e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ /dev/null
@@ -1,2850 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.agent;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.types.Endpoint;
-import org.apache.hadoop.registry.client.types.ProtocolTypes;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.StatusKeys;
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTreeOperations;
-import org.apache.slider.core.conf.MapOperations;
-import org.apache.slider.core.exceptions.BadConfigException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.CommandLineBuilder;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.ConfigUtils;
-import org.apache.slider.core.registry.docstore.ExportEntry;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedExports;
-import org.apache.slider.core.registry.info.CustomRegistryConstants;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.providers.MonitorDetail;
-import org.apache.slider.providers.ProviderCore;
-import org.apache.slider.providers.ProviderRole;
-import org.apache.slider.providers.ProviderUtils;
-import 
org.apache.slider.providers.agent.application.metadata.AbstractComponent;
-import org.apache.slider.providers.agent.application.metadata.Application;
-import org.apache.slider.providers.agent.application.metadata.CommandOrder;
-import org.apache.slider.providers.agent.application.metadata.CommandScript;
-import org.apache.slider.providers.agent.application.metadata.Component;
-import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
-import org.apache.slider.providers.agent.application.metadata.ComponentExport;
-import 
org.apache.slider.providers.agent.application.metadata.ComponentsInAddonPackage;
-import org.apache.slider.providers.agent.application.metadata.ConfigFile;
-import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
-import 

[43/50] [abbrv] hadoop git commit: YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He

[jianhe]

YARN-5967. Fix slider core module findbugs warnings. Contributed by Jian He


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2f380fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2f380fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2f380fd

Branch: refs/heads/yarn-native-services
Commit: e2f380fded90dc8f6e2ef735ced844d5e750ba18
Parents: 9c1c024
Author: Billie Rinaldi 
Authored: Mon Dec 19 12:09:53 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../dev-support/findbugs-exclude.xml|  96 +
 .../dev-support/findbugs-exclude.xml|  20 -
 .../hadoop-yarn-slider-core/pom.xml |   5 +
 .../apache/hadoop/security/KerberosDiags.java   |   5 +-
 .../slider/api/proto/RestTypeMarshalling.java   | 283 -
 .../slider/api/types/ComponentInformation.java  |   2 +-
 .../slider/api/types/RestTypeMarshalling.java   | 284 +
 .../org/apache/slider/client/SliderClient.java  |  56 +--
 .../apache/slider/client/TokensOperation.java   |   9 +-
 .../client/ipc/SliderClusterOperations.java |   2 +-
 .../rest/SliderApplicationApiRestClient.java|  14 -
 .../org/apache/slider/common/SliderKeys.java|   8 +-
 .../common/params/AbstractActionArgs.java   |   5 +-
 .../AbstractClusterBuildingActionArgs.java  |   1 -
 .../apache/slider/common/params/CommonArgs.java |   9 +-
 .../apache/slider/common/tools/Comparators.java |  17 +-
 .../slider/common/tools/ConfigHelper.java   |   2 +-
 .../slider/common/tools/CoreFileSystem.java |  31 --
 .../apache/slider/common/tools/SliderUtils.java | 102 +
 .../slider/core/buildutils/InstanceBuilder.java |   1 -
 .../apache/slider/core/conf/AggregateConf.java  |   2 +-
 .../org/apache/slider/core/conf/ConfTree.java   |  15 +-
 .../core/persist/AppDefinitionPersister.java|  15 +-
 .../core/registry/retrieve/AMWebClient.java |  51 ---
 .../core/restclient/HttpOperationResponse.java  |  34 --
 .../restclient/UrlConnectionOperations.java | 120 --
 .../slider/core/zk/MiniZooKeeperCluster.java|  27 +-
 .../apache/slider/core/zk/ZKIntegration.java|  20 +-
 .../apache/slider/providers/ProviderUtils.java  |  21 +-
 .../providers/docker/DockerProviderService.java |  12 +-
 .../server/appmaster/management/BoolMetric.java |  15 -
 .../management/MetricsAndMonitoring.java|  25 --
 .../management/RangeLimitedCounter.java |  85 
 .../appmaster/management/RecordedEvent.java |  58 ---
 .../server/appmaster/rpc/SliderIPCService.java  |   2 +-
 .../server/appmaster/state/NodeInstance.java|   2 +-
 .../state/OutstandingRequestTracker.java|   2 +-
 .../server/appmaster/state/RoleStatus.java  |   4 -
 .../appmaster/web/rest/InsecureAmFilter.java|   1 -
 .../web/rest/publisher/PublisherResource.java   |   5 +-
 .../appmaster/web/view/ContainerStatsBlock.java |   3 +-
 .../server/servicemonitor/ProbeStatus.java  |   1 +
 .../utility/AbstractSliderLaunchedService.java  |   2 -
 .../services/workflow/LongLivedProcess.java |   5 +-
 .../slider/common/tools/TestSliderUtils.java|   2 +-
 .../slider/test/MiniZooKeeperCluster.java   | 395 ---
 46 files changed, 501 insertions(+), 1375 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2f380fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
new file mode 100644
index 000..d253762
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/dev-support/findbugs-exclude.xml
@@ -0,0 +1,96 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

[31/50] [abbrv] hadoop git commit: YARN-5883 Avoid or eliminate expensive YARN get all applications call. Contributed by Gour Saha

[jianhe]

YARN-5883 Avoid or eliminate expensive YARN get all applications call. 
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/528626b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/528626b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/528626b8

Branch: refs/heads/yarn-native-services
Commit: 528626b85f087c3454dad253fe5f5fa42cb6a8f3
Parents: cf5ffbb
Author: Billie Rinaldi 
Authored: Tue Nov 29 13:14:45 2016 -0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/client/SliderClient.java  |  6 +-
 .../slider/client/SliderYarnClientImpl.java | 94 +---
 .../apache/slider/common/tools/SliderUtils.java | 37 +++-
 .../slider/core/registry/YarnAppListClient.java | 21 -
 .../providers/AbstractClientProvider.java   | 28 +-
 .../providers/docker/DockerClientProvider.java  |  8 ++
 .../servicemonitor/YarnApplicationProbe.java| 12 +--
 7 files changed, 160 insertions(+), 46 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/528626b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index ea10ed0..3f2df0a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -198,6 +198,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -2083,7 +2084,7 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 
 // add the tags if available
 Set applicationTags = provider.getApplicationTags(sliderFileSystem,
-appOperations);
+appOperations, clustername);
 
 Credentials credentials = null;
 if (clusterSecure) {
@@ -3031,9 +3032,10 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
 appstate.ordinal() < YarnApplicationState.FINISHED.ordinal();
 } else {
   // scan for instance in single --state state
-  List userInstances = 
yarnClient.listDeployedInstances("");
   state = state.toUpperCase(Locale.ENGLISH);
   YarnApplicationState desiredState = extractYarnApplicationState(state);
+  List userInstances = yarnClient
+  .listDeployedInstances("", EnumSet.of(desiredState), name);
   ApplicationReport foundInstance =
   yarnClient.findAppInInstanceList(userInstances, name, desiredState);
   if (foundInstance != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/528626b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index d471cdb..258ef31 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -53,6 +53,8 @@ import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -115,10 +117,59 @@ public class SliderYarnClientImpl extends YarnClientImpl {
*/
   public List listDeployedInstances(String user)
 throws YarnException, 

[01/50] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi [Forced Update!]

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/yarn-native-services 7e7b54855 -> 732c5714e (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb860960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
new file mode 100644
index 000..bebb5f0
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.docker;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.slider.api.ClusterDescription;
+import org.apache.slider.api.ClusterNode;
+import org.apache.slider.api.OptionKeys;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.conf.MapOperations;
+import org.apache.slider.core.exceptions.SliderException;
+import org.apache.slider.core.launch.CommandLineBuilder;
+import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.core.registry.docstore.ConfigFormat;
+import org.apache.slider.core.registry.docstore.ConfigUtils;
+import org.apache.slider.core.registry.docstore.ExportEntry;
+import org.apache.slider.providers.AbstractProviderService;
+import org.apache.slider.providers.ProviderCore;
+import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.server.appmaster.state.RoleInstance;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Scanner;
+
+public class DockerProviderService extends AbstractProviderService implements
+ProviderCore,
+DockerKeys,
+SliderKeys {
+
+  protected static final Logger log =
+  LoggerFactory.getLogger(DockerProviderService.class);
+  private static final ProviderUtils providerUtils = new ProviderUtils(log);
+  private static final String EXPORT_GROUP = "quicklinks";
+  private static final String APPLICATION_TAG = "application";
+
+  private String clusterName = null;
+  private SliderFileSystem fileSystem = null;
+
+  protected DockerProviderService() {
+super("DockerProviderService");
+  }
+
+  @Override
+  public List getRoles() {
+return Collections.emptyList();
+  }
+
+  @Override
+  public boolean isSupportedRole(String role) {
+return true;
+  }
+
+  @Override
+  public void validateInstanceDefinition(AggregateConf instanceDefinition)
+  throws SliderException {
+  }
+
+  private String getClusterName() {
+if (SliderUtils.isUnset(clusterName)) {
+  clusterName = 

[07/50] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

YARN-5610. Initial code for native services REST API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58ad73fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58ad73fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58ad73fd

Branch: refs/heads/yarn-native-services
Commit: 58ad73fdf869d7c917f73fdda3b2717eb2d99d62
Parents: 742569f
Author: Jian He 
Authored: Tue Oct 11 11:36:57 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 hadoop-project/pom.xml  |   20 +
 .../dev-support/findbugs-exclude.xml|   20 +
 .../hadoop-yarn-services-api/pom.xml|  225 +++
 .../yarn/services/api/ApplicationApi.java   |   38 +
 .../api/impl/ApplicationApiService.java | 1527 ++
 .../yarn/services/resource/Application.java |  452 ++
 .../services/resource/ApplicationState.java |   25 +
 .../services/resource/ApplicationStatus.java|  147 ++
 .../hadoop/yarn/services/resource/Artifact.java |  155 ++
 .../yarn/services/resource/BaseResource.java|   48 +
 .../yarn/services/resource/Component.java   |  377 +
 .../yarn/services/resource/ConfigFile.java  |  190 +++
 .../yarn/services/resource/Configuration.java   |  147 ++
 .../yarn/services/resource/Container.java   |  256 +++
 .../yarn/services/resource/ContainerState.java  |   25 +
 .../hadoop/yarn/services/resource/Error.java|  125 ++
 .../yarn/services/resource/PlacementPolicy.java |   97 ++
 .../yarn/services/resource/ReadinessCheck.java  |  161 ++
 .../hadoop/yarn/services/resource/Resource.java |  149 ++
 .../yarn/services/utils/RestApiConstants.java   |   66 +
 .../services/utils/RestApiErrorMessages.java|   79 +
 .../services/webapp/ApplicationApiWebApp.java   |  127 ++
 .../src/main/resources/log4j-server.properties  |   76 +
 .../resources/webapps/services-rest-api/app |   16 +
 .../src/main/scripts/run_rest_service.sh|   28 +
 .../src/main/webapp/WEB-INF/web.xml |   36 +
 .../api/impl/TestApplicationApiService.java |  232 +++
 .../hadoop-yarn-applications/pom.xml|2 +-
 28 files changed, 4845 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/58ad73fd/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d3579a1..24ba138 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -130,6 +130,9 @@
 
 ${project.version}
+
+1.5.4
+
1.4
   
 
   
@@ -1245,6 +1248,23 @@
   kerb-simplekdc
   1.0.0-RC2
 
+
+  
+io.swagger
+swagger-annotations
+${swagger-annotations-version}
+  
+  
+com.fasterxml.jackson.jaxrs
+jackson-jaxrs-json-provider
+${jackson2.version}
+  
+  
+org.apache.maven.doxia
+doxia-module-markdown
+${maven-doxia-module-markdown.version}
+  
+
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58ad73fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
new file mode 100644
index 000..b89146a
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/dev-support/findbugs-exclude.xml
@@ -0,0 +1,20 @@
+
+
+
+
+

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58ad73fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
new file mode 100644
index 000..78b7855
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/pom.xml
@@ -0,0 +1,225 @@
+
+http://maven.apache.org/POM/4.0.0; 
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; 
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 
http://maven.apache.org/maven-v4_0_0.xsd;>
+  4.0.0
+  
+org.apache.hadoop
+hadoop-yarn-applications
+3.0.0-alpha2-SNAPSHOT
+  
+  org.apache.hadoop
+  hadoop-yarn-services-api
+  Apache Hadoop YARN Services API
+  

[15/50] [abbrv] hadoop git commit: YARN-5680. Add 2 new fields in Slider status output - image-name and is-privileged-container. Contributed by Billie Rinaldi

[jianhe]

YARN-5680. Add 2 new fields in Slider status output - image-name and 
is-privileged-container. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8500ad4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8500ad4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8500ad4e

Branch: refs/heads/yarn-native-services
Commit: 8500ad4eefb4fb821de7339ea47da2d2bbfa4da0
Parents: 5e5a4de
Author: Gour Saha 
Authored: Tue Oct 25 20:00:27 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/providers/docker/DockerKeys.java  |  1 +
 .../slider/providers/docker/DockerProviderService.java  |  2 +-
 .../apache/slider/server/appmaster/state/AppState.java  | 12 
 3 files changed, 14 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8500ad4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
index 40b73a2..0e1d288 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
@@ -26,6 +26,7 @@ public interface DockerKeys {
   String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
 
   String DEFAULT_DOCKER_NETWORK = "bridge";
+  Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
 
   String OUT_FILE = "stdout.txt";
   String ERR_FILE = "stderr.txt";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8500ad4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index af36620..cc319ee 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -131,7 +131,7 @@ public class DockerProviderService extends 
AbstractProviderService implements
 launcher.setDockerNetwork(appConf.getComponentOpt(roleGroup, 
DOCKER_NETWORK,
 DEFAULT_DOCKER_NETWORK));
 launcher.setRunPrivilegedContainer(appConf.getComponentOptBool(roleGroup,
-DOCKER_USE_PRIVILEGED, false));
+DOCKER_USE_PRIVILEGED, DEFAULT_DOCKER_USE_PRIVILEGED));
 
 // Set the environment
 Map standardTokens = providerUtils.getStandardTokenMap(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8500ad4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 49e7b78..6db375d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -91,6 +91,9 @@ 

[05/50] [abbrv] hadoop git commit: YARN-5610. Initial code for native services REST API. Contributed by Gour Saha

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58ad73fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
new file mode 100644
index 000..a3780cc
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Resource.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.services.resource;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+
+import java.util.Objects;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Resource determines the amount of resources (vcores, memory, network, etc.)
+ * usable by a container. This field determines the resource to be applied for
+ * all the containers of a component or application. The resource specified at
+ * the app (or global) level can be overriden at the component level. Only one
+ * of profile OR cpu  memory are exepected. It raises a validation
+ * exception otherwise.
+ **/
+
+@ApiModel(description = "Resource determines the amount of resources (vcores, 
memory, network, etc.) usable by a container. This field determines the 
resource to be applied for all the containers of a component or application. 
The resource specified at the app (or global) level can be overriden at the 
component level. Only one of profile OR cpu & memory are exepected. It raises a 
validation exception otherwise.")
+@javax.annotation.Generated(value = "class 
io.swagger.codegen.languages.JavaClientCodegen", date = 
"2016-06-02T08:15:05.615-07:00")
+public class Resource extends BaseResource {
+  private static final long serialVersionUID = -6431667797380250037L;
+
+  private String profile = null;
+  private Integer cpus = null;
+  private String memory = null;
+
+  /**
+   * Each resource profile has a unique id which is associated with a
+   * cluster-level predefined memory, cpus, etc.
+   **/
+  public Resource profile(String profile) {
+this.profile = profile;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Each resource profile has a 
unique id which is associated with a cluster-level predefined memory, cpus, 
etc.")
+  @JsonProperty("profile")
+  public String getProfile() {
+return profile;
+  }
+
+  public void setProfile(String profile) {
+this.profile = profile;
+  }
+
+  /**
+   * Amount of vcores allocated to each container (optional but overrides cpus
+   * in profile if specified).
+   **/
+  public Resource cpus(Integer cpus) {
+this.cpus = cpus;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Amount of vcores allocated to 
each container (optional but overrides cpus in profile if specified).")
+  @JsonProperty("cpus")
+  public Integer getCpus() {
+return cpus;
+  }
+
+  public void setCpus(Integer cpus) {
+this.cpus = cpus;
+  }
+
+  /**
+   * Amount of memory allocated to each container (optional but overrides 
memory
+   * in profile if specified). Currently accepts only an integer value and
+   * default unit is in MB.
+   **/
+  public Resource memory(String memory) {
+this.memory = memory;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "Amount of memory allocated to 
each container (optional but overrides memory in profile if specified). 
Currently accepts only an integer value and default unit is in MB.")
+  @JsonProperty("memory")
+  public String getMemory() {
+return memory;
+  }
+
+  public void setMemory(String memory) {
+this.memory = memory;
+  }
+
+  @Override
+  public boolean equals(java.lang.Object o) {
+if (this == o) {
+  return true;
+}
+if (o == null || getClass() != 

[11/50] [abbrv] hadoop git commit: YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha

[jianhe]

YARN-5729. Bug fixes for the service Rest API. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d7d4f7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d7d4f7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d7d4f7f

Branch: refs/heads/yarn-native-services
Commit: 6d7d4f7feb7d873728e0bd161846c78789ec51af
Parents: 2034e9b
Author: Jian He 
Authored: Fri Oct 14 13:47:38 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../services/api/impl/ApplicationApiService.java| 16 ++--
 .../hadoop/yarn/services/resource/Application.java  |  6 +++---
 .../hadoop/yarn/services/resource/Artifact.java |  4 +++-
 .../hadoop/yarn/services/resource/Component.java|  4 +++-
 .../hadoop/yarn/services/resource/ConfigFile.java   |  4 +++-
 .../yarn/services/resource/Configuration.java   |  4 +++-
 .../hadoop/yarn/services/resource/Container.java|  6 +++---
 .../yarn/services/resource/PlacementPolicy.java |  4 +++-
 .../yarn/services/resource/ReadinessCheck.java  |  4 +++-
 .../hadoop/yarn/services/resource/Resource.java |  2 +-
 10 files changed, 39 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d7d4f7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
index 0a62629..21cf113 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/api/impl/ApplicationApiService.java
@@ -276,7 +276,7 @@ public class ApplicationApiService implements 
ApplicationApi {
 
 // If it is a simple app with no components, then create a default 
component
 if (application.getComponents() == null) {
-  application.setComponents(getDefaultComponentAsList());
+  application.setComponents(getDefaultComponentAsList(application));
 }
 
 // Application lifetime if not specified, is set to unlimited lifetime
@@ -1029,7 +1029,8 @@ public class ApplicationApiService implements 
ApplicationApi {
 // end-users point of view, is out of scope of the REST API. Also, this
 // readiness has nothing to do with readiness-check defined at the 
component
 // level (which is used for dependency resolution of component DAG).
-if (totalNumberOfIpAssignedContainers == 
totalExpectedNumberOfRunningContainers) {
+if (totalNumberOfIpAssignedContainers
+.longValue() == totalExpectedNumberOfRunningContainers.longValue()) {
   app.setState(ApplicationState.READY);
 }
 logger.info("Application = {}", app);
@@ -1389,6 +1390,17 @@ public class ApplicationApiService implements 
ApplicationApi {
 return Response.status(Status.NO_CONTENT).build();
   }
 
+  // create default component and initialize with app level global values
+  private List getDefaultComponentAsList(Application app) {
+List comps = getDefaultComponentAsList();
+Component comp = comps.get(0);
+comp.setArtifact(app.getArtifact());
+comp.setResource(app.getResource());
+comp.setNumberOfContainers(app.getNumberOfContainers());
+comp.setLaunchCommand(app.getLaunchCommand());
+return comps;
+  }
+
   private List getDefaultComponentAsList() {
 Component comp = new Component();
 comp.setName(DEFAULT_COMPONENT_NAME);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d7d4f7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index 719bf95..ed65ad2 100644
--- 

[14/50] [abbrv] hadoop git commit: YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by Billie Rinaldi

[jianhe]

YARN-5701. Fix issues in yarn native services apps-of-apps. Contributed by 
Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1cf51d4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1cf51d4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1cf51d4f

Branch: refs/heads/yarn-native-services
Commit: 1cf51d4fd6922ac64bea9cbfe1c57c93e93cce75
Parents: 7cca292
Author: Jian He 
Authored: Sun Oct 16 17:01:09 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../org/apache/slider/client/SliderClient.java  | 72 
 .../slider/core/buildutils/InstanceBuilder.java |  4 +
 .../apache/slider/providers/ProviderUtils.java  | 62 --
 .../providers/docker/DockerClientProvider.java  |  4 +-
 .../providers/docker/DockerProviderService.java | 87 
 5 files changed, 164 insertions(+), 65 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1cf51d4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 2840c4b..94e51e5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -178,6 +178,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
+import java.io.Console;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -918,57 +919,56 @@ public class SliderClient extends 
AbstractSliderLaunchedService implements RunSe
   return;
 }
 
-BufferedReader br = null;
-try {
-  for (Entry cred : tree.credentials.entrySet()) {
-String provider = cred.getKey()
-.replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
-.replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
-List aliases = cred.getValue();
-if (aliases == null || aliases.isEmpty()) {
-  continue;
-}
-Configuration c = new Configuration(conf);
-c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
-CredentialProvider credentialProvider = 
CredentialProviderFactory.getProviders(c).get(0);
-Set existingAliases = new 
HashSet<>(credentialProvider.getAliases());
-for (String alias : aliases) {
-  if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
-log.info("Credentials for " + alias + " found in " + provider);
-  } else {
-if (br == null) {
-  br = new BufferedReader(new InputStreamReader(System.in));
-}
-char[] pass = readPassword(alias, br);
-credentialProvider.createCredentialEntry(alias, pass);
-credentialProvider.flush();
-Arrays.fill(pass, ' ');
+Console console = System.console();
+for (Entry cred : tree.credentials.entrySet()) {
+  String provider = cred.getKey()
+  .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
+  .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
+  List aliases = cred.getValue();
+  if (aliases == null || aliases.isEmpty()) {
+continue;
+  }
+  Configuration c = new Configuration(conf);
+  c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
+  CredentialProvider credentialProvider = 
CredentialProviderFactory.getProviders(c).get(0);
+  Set existingAliases = new 
HashSet<>(credentialProvider.getAliases());
+  for (String alias : aliases) {
+if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
+  log.info("Credentials for " + alias + " found in " + provider);
+} else {
+  if (console == null) {
+throw new IOException("Unable to input password for " + alias +
+" because System.console() is null; provider " + provider +
+" must be populated manually");
   }
+  char[] pass = readPassword(alias, console);
+  

[09/50] [abbrv] hadoop git commit: Addendum patch for YARN-5610. Contributed by Gour Saha

[jianhe]

Addendum patch for YARN-5610. Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2420ab9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2420ab9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2420ab9

Branch: refs/heads/yarn-native-services
Commit: b2420ab9451393d7d2196a89ca764a92a6d77323
Parents: b47fb5d
Author: Jian He 
Authored: Wed Oct 12 13:33:09 2016 -0700
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../yarn/services/resource/Application.java | 44 ++--
 .../services/resource/ApplicationState.java |  5 +++
 .../services/resource/ApplicationStatus.java|  8 ++--
 .../hadoop/yarn/services/resource/Artifact.java |  4 +-
 .../yarn/services/resource/Component.java   | 16 +++
 .../yarn/services/resource/Container.java   | 15 ---
 .../yarn/services/resource/ReadinessCheck.java  |  6 +--
 7 files changed, 54 insertions(+), 44 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2420ab9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
index cfcae95..719bf95 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/Application.java
@@ -48,8 +48,8 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
 public class Application extends BaseResource {
   private static final long serialVersionUID = -4491694636566094885L;
 
-  private String id = null;
   private String name = null;
+  private String id = null;
   private Artifact artifact = null;
   private Resource resource = null;
   private String launchCommand = null;
@@ -63,25 +63,7 @@ public class Application extends BaseResource {
   private List containers = new ArrayList<>();
   private ApplicationState state = null;
   private Map quicklinks = null;
-  private String queue;
-
-  /**
-   * A unique application id.
-   **/
-  public Application id(String id) {
-this.id = id;
-return this;
-  }
-
-  @ApiModelProperty(example = "null", required = true, value = "A unique 
application id.")
-  @JsonProperty("id")
-  public String getId() {
-return id;
-  }
-
-  public void setId(String id) {
-this.id = id;
-  }
+  private String queue = null;
 
   /**
* A unique application name.
@@ -102,6 +84,24 @@ public class Application extends BaseResource {
   }
 
   /**
+   * A unique application id.
+   **/
+  public Application id(String id) {
+this.id = id;
+return this;
+  }
+
+  @ApiModelProperty(example = "null", value = "A unique application id.")
+  @JsonProperty("id")
+  public String getId() {
+return id;
+  }
+
+  public void setId(String id) {
+this.id = id;
+  }
+
+  /**
* Artifact of single-component applications. Mandatory if components
* attribute is not specified.
**/
@@ -423,8 +423,8 @@ public class Application extends BaseResource {
 sb.append("numberOfRunningContainers: ")
 .append(toIndentedString(numberOfRunningContainers)).append("\n");
 sb.append("lifetime: 
").append(toIndentedString(lifetime)).append("\n");
-sb.append("placementPolicy: ")
-.append(toIndentedString(placementPolicy)).append("\n");
+sb.append("placementPolicy: 
").append(toIndentedString(placementPolicy))
+.append("\n");
 sb.append("components: ").append(toIndentedString(components))
 .append("\n");
 sb.append("configuration: ").append(toIndentedString(configuration))

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2420ab9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/services/resource/ApplicationState.java
 

[03/50] [abbrv] hadoop git commit: YARN-5505. Create an agent-less docker provider in the native-services framework. Contributed by Billie Rinaldi

[jianhe]

YARN-5505. Create an agent-less docker provider in the native-services 
framework. Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb860960
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb860960
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb860960

Branch: refs/heads/yarn-native-services
Commit: bb86096026dfd43f2544c0e870857a01e587b470
Parents: 4224575
Author: Jian He 
Authored: Thu Sep 1 22:38:42 2016 +0800
Committer: Jian He 
Committed: Thu Feb 9 16:58:05 2017 -0800

--
 .../java/org/apache/slider/api/OptionKeys.java  |   15 +-
 .../org/apache/slider/client/SliderClient.java  |   17 +-
 .../org/apache/slider/common/SliderKeys.java|   22 +-
 .../apache/slider/common/tools/SliderUtils.java |4 +
 .../slider/core/launch/AbstractLauncher.java|   18 +-
 .../PublishedConfigurationOutputter.java|6 +-
 .../providers/AbstractClientProvider.java   |4 +-
 .../providers/AbstractProviderService.java  |   22 +-
 .../slider/providers/ProviderService.java   |   12 +-
 .../apache/slider/providers/ProviderUtils.java  | 1391 ++
 .../providers/agent/AgentClientProvider.java|   36 +-
 .../slider/providers/agent/AgentKeys.java   |   12 +-
 .../providers/agent/AgentProviderService.java   |  705 ++---
 .../providers/docker/DockerClientProvider.java  |   96 ++
 .../slider/providers/docker/DockerKeys.java |   32 +
 .../providers/docker/DockerProviderFactory.java |   43 +
 .../providers/docker/DockerProviderService.java |  355 +
 .../slideram/SliderAMProviderService.java   |4 -
 .../server/appmaster/SliderAppMaster.java   |   39 +-
 .../main/resources/org/apache/slider/slider.xml |4 +
 .../slider/providers/docker/appConfig.json  |   42 +
 .../slider/providers/docker/resources.json  |   16 +
 .../slider/providers/docker/test.template   |   16 +
 23 files changed, 1971 insertions(+), 940 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb860960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
index a035a99..434b1d9 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/OptionKeys.java
@@ -41,7 +41,20 @@ public interface OptionKeys extends InternalKeys {
* Prefix for site.xml options: {@value}
*/
   String SITE_XML_PREFIX = "site.";
-
+  /**
+   * Prefix for config file options: {@value}
+   */
+  String CONF_FILE_PREFIX = "conf.";
+  /**
+   * Prefix for package options: {@value}
+   */
+  String PKG_FILE_PREFIX = "pkg.";
+  /**
+   * Prefix for export options: {@value}
+   */
+  String EXPORT_PREFIX = "export.";
+  String TYPE_SUFFIX = ".type";
+  String NAME_SUFFIX = ".name";
 
   /**
* Zookeeper quorum host list: {@value}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb860960/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3129f6f..5096bb7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -151,7 +151,6 @@ import org.apache.slider.core.registry.YarnAppListClient;
 import org.apache.slider.core.registry.docstore.ConfigFormat;
 import org.apache.slider.core.registry.docstore.PublishedConfigSet;
 import org.apache.slider.core.registry.docstore.PublishedConfiguration;

hadoop git commit: YARN-6144. FairScheduler: preempted resources can become negative. (Miklos Szegedi via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5de6f1b79 -> e259a818a


YARN-6144. FairScheduler: preempted resources can become negative. (Miklos 
Szegedi via kasha)

(cherry picked from commit 08f93978f3ec724b24a93d7ef538f158da75802f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e259a818
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e259a818
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e259a818

Branch: refs/heads/branch-2
Commit: e259a818adb9c95bf2966f0898defc4a4d41bb01
Parents: 5de6f1b
Author: Karthik Kambatla 
Authored: Thu Feb 9 14:45:04 2017 -0800
Committer: Karthik Kambatla 
Committed: Thu Feb 9 14:45:21 2017 -0800

--
 .../scheduler/fair/FSAppAttempt.java| 16 +---
 .../scheduler/fair/TestFairScheduler.java   | 16 
 2 files changed, 25 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e259a818/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 5f5d14c..9333caa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -542,18 +542,20 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   }
 
   void trackContainerForPreemption(RMContainer container) {
-containersToPreempt.add(container);
-synchronized (preemptedResources) {
-  Resources.addTo(preemptedResources, container.getAllocatedResource());
+if (containersToPreempt.add(container)) {
+  synchronized (preemptedResources) {
+Resources.addTo(preemptedResources, container.getAllocatedResource());
+  }
 }
   }
 
   private void untrackContainerForPreemption(RMContainer container) {
-synchronized (preemptedResources) {
-  Resources.subtractFrom(preemptedResources,
-  container.getAllocatedResource());
+if (containersToPreempt.remove(container)) {
+  synchronized (preemptedResources) {
+Resources.subtractFrom(preemptedResources,
+container.getAllocatedResource());
+  }
 }
-containersToPreempt.remove(container);
   }
 
   Set getPreemptionContainers() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e259a818/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 5c86f98..cc68b36 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3640,6 +3640,8 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 scheduler.handle(updateEvent);
 assertEquals("Application1's AM should be finished",
 0, app1.getLiveContainers().size());
+assertEquals("Finished application usage should be none",
+Resources.none(), app1.getResourceUsage());
 assertEquals("Application3's AM should be running",
 1, app3.getLiveContainers().size());
 assertEquals("Application3's AM requests 1024 MB memory",
@@ -3658,6 +3660,8 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 0, app4.getAMResource().getMemorySize());
 

hadoop git commit: YARN-6144. FairScheduler: preempted resources can become negative. (Miklos Szegedi via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 0c01cf579 -> 08f93978f


YARN-6144. FairScheduler: preempted resources can become negative. (Miklos 
Szegedi via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08f93978
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08f93978
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08f93978

Branch: refs/heads/trunk
Commit: 08f93978f3ec724b24a93d7ef538f158da75802f
Parents: 0c01cf5
Author: Karthik Kambatla 
Authored: Thu Feb 9 14:45:04 2017 -0800
Committer: Karthik Kambatla 
Committed: Thu Feb 9 14:45:04 2017 -0800

--
 .../scheduler/fair/FSAppAttempt.java| 16 +---
 .../scheduler/fair/TestFairScheduler.java   | 16 
 2 files changed, 25 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f93978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0715e3a..9e57fa7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -543,18 +543,20 @@ public class FSAppAttempt extends 
SchedulerApplicationAttempt
   }
 
   void trackContainerForPreemption(RMContainer container) {
-containersToPreempt.add(container);
-synchronized (preemptedResources) {
-  Resources.addTo(preemptedResources, container.getAllocatedResource());
+if (containersToPreempt.add(container)) {
+  synchronized (preemptedResources) {
+Resources.addTo(preemptedResources, container.getAllocatedResource());
+  }
 }
   }
 
   private void untrackContainerForPreemption(RMContainer container) {
-synchronized (preemptedResources) {
-  Resources.subtractFrom(preemptedResources,
-  container.getAllocatedResource());
+if (containersToPreempt.remove(container)) {
+  synchronized (preemptedResources) {
+Resources.subtractFrom(preemptedResources,
+container.getAllocatedResource());
+  }
 }
-containersToPreempt.remove(container);
   }
 
   Set getPreemptionContainers() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f93978/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index b1e412b..ce32459 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3649,6 +3649,8 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 scheduler.handle(updateEvent);
 assertEquals("Application1's AM should be finished",
 0, app1.getLiveContainers().size());
+assertEquals("Finished application usage should be none",
+Resources.none(), app1.getResourceUsage());
 assertEquals("Application3's AM should be running",
 1, app3.getLiveContainers().size());
 assertEquals("Application3's AM requests 1024 MB memory",
@@ -3667,6 +3669,8 @@ public class TestFairScheduler extends 
FairSchedulerTestBase {
 0, app4.getAMResource().getMemorySize());
 assertEquals("Application4's AM should not be running",
 0, 

hadoop git commit: HADOOP-14033. Reduce fair call queue lock contention. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 47bbe431b -> 6b3443fbf


HADOOP-14033. Reduce fair call queue lock contention. Contributed by Daryn 
Sharp.

(cherry picked from commit 0c01cf57987bcc7a17154a3538960b67f625a9e5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b3443fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b3443fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b3443fb

Branch: refs/heads/branch-2.8
Commit: 6b3443fbf5bb38e6890b6d99d63ab9436f06fc94
Parents: 47bbe43
Author: Kihwal Lee 
Authored: Thu Feb 9 16:20:48 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 16:20:48 2017 -0600

--
 .../org/apache/hadoop/ipc/FairCallQueue.java| 167 ++-
 1 file changed, 51 insertions(+), 116 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b3443fb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 77a9d65..820f24c 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -27,8 +27,7 @@ import java.util.AbstractQueue;
 import java.util.HashMap;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.Condition;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -55,16 +54,15 @@ public class FairCallQueue extends 
AbstractQueue
   /* The queues */
   private final ArrayList queues;
 
-  /* Read locks */
-  private final ReentrantLock takeLock = new ReentrantLock();
-  private final Condition notEmpty = takeLock.newCondition();
+  /* Track available permits for scheduled objects.  All methods that will
+   * mutate a subqueue must acquire or release a permit on the semaphore.
+   * A semaphore is much faster than an exclusive lock because producers do
+   * not contend with consumers and consumers do not block other consumers
+   * while polling.
+   */
+  private final Semaphore semaphore = new Semaphore(0);
   private void signalNotEmpty() {
-takeLock.lock();
-try {
-  notEmpty.signal();
-} finally {
-  takeLock.unlock();
-}
+semaphore.release();
   }
 
   /* Multiplexer picks which queue to draw from */
@@ -112,28 +110,25 @@ public class FairCallQueue extends 
AbstractQueue
   }
 
   /**
-   * Returns the first non-empty queue with equal to startIdx, or
-   * or scans from highest to lowest priority queue.
+   * Returns an element first non-empty queue equal to the priority returned
+   * by the multiplexer or scans from highest to lowest priority queue.
+   *
+   * Caller must always acquire a semaphore permit before invoking.
*
-   * @param startIdx the queue number to start searching at
* @return the first non-empty queue with less priority, or null if
* everything was empty
*/
-  private BlockingQueue getFirstNonEmptyQueue(int startIdx) {
-BlockingQueue queue = this.queues.get(startIdx);
-if (queue.size() != 0) {
-  return queue;
-}
-final int numQueues = this.queues.size();
-for(int i=0; i < numQueues; i++) {
-  queue = this.queues.get(i);
-  if (queue.size() != 0) {
-return queue;
+  private E removeNextElement() {
+int priority = multiplexer.getAndAdvanceCurrentIndex();
+E e = queues.get(priority).poll();
+if (e == null) {
+  for (int idx = 0; e == null && idx < queues.size(); idx++) {
+e = queues.get(idx).poll();
   }
 }
-
-// All queues were empty
-return null;
+// guaranteed to find an element if caller acquired permit.
+assert e != null : "consumer didn't acquire semaphore!";
+return e;
   }
 
   /* AbstractQueue and BlockingQueue methods */
@@ -184,9 +179,9 @@ public class FairCallQueue extends 
AbstractQueue
 int priorityLevel = e.getPriorityLevel();
 BlockingQueue q = this.queues.get(priorityLevel);
 boolean ret = q.offer(e, timeout, unit);
-
-signalNotEmpty();
-
+if (ret) {
+  signalNotEmpty();
+}
 return ret;
   }
 
@@ -195,72 +190,21 @@ public class FairCallQueue extends 
AbstractQueue
 int priorityLevel = e.getPriorityLevel();
 BlockingQueue q = this.queues.get(priorityLevel);
 boolean ret = q.offer(e);
-
-

hadoop git commit: HADOOP-14033. Reduce fair call queue lock contention. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 41361ec39 -> 5de6f1b79


HADOOP-14033. Reduce fair call queue lock contention. Contributed by Daryn 
Sharp.

(cherry picked from commit 0c01cf57987bcc7a17154a3538960b67f625a9e5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5de6f1b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5de6f1b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5de6f1b7

Branch: refs/heads/branch-2
Commit: 5de6f1b79140499452eef2356e67f74750dacb2a
Parents: 41361ec
Author: Kihwal Lee 
Authored: Thu Feb 9 16:20:06 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 16:20:06 2017 -0600

--
 .../org/apache/hadoop/ipc/FairCallQueue.java| 167 ++-
 1 file changed, 51 insertions(+), 116 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5de6f1b7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 77a9d65..820f24c 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -27,8 +27,7 @@ import java.util.AbstractQueue;
 import java.util.HashMap;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.Condition;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -55,16 +54,15 @@ public class FairCallQueue extends 
AbstractQueue
   /* The queues */
   private final ArrayList queues;
 
-  /* Read locks */
-  private final ReentrantLock takeLock = new ReentrantLock();
-  private final Condition notEmpty = takeLock.newCondition();
+  /* Track available permits for scheduled objects.  All methods that will
+   * mutate a subqueue must acquire or release a permit on the semaphore.
+   * A semaphore is much faster than an exclusive lock because producers do
+   * not contend with consumers and consumers do not block other consumers
+   * while polling.
+   */
+  private final Semaphore semaphore = new Semaphore(0);
   private void signalNotEmpty() {
-takeLock.lock();
-try {
-  notEmpty.signal();
-} finally {
-  takeLock.unlock();
-}
+semaphore.release();
   }
 
   /* Multiplexer picks which queue to draw from */
@@ -112,28 +110,25 @@ public class FairCallQueue extends 
AbstractQueue
   }
 
   /**
-   * Returns the first non-empty queue with equal to startIdx, or
-   * or scans from highest to lowest priority queue.
+   * Returns an element first non-empty queue equal to the priority returned
+   * by the multiplexer or scans from highest to lowest priority queue.
+   *
+   * Caller must always acquire a semaphore permit before invoking.
*
-   * @param startIdx the queue number to start searching at
* @return the first non-empty queue with less priority, or null if
* everything was empty
*/
-  private BlockingQueue getFirstNonEmptyQueue(int startIdx) {
-BlockingQueue queue = this.queues.get(startIdx);
-if (queue.size() != 0) {
-  return queue;
-}
-final int numQueues = this.queues.size();
-for(int i=0; i < numQueues; i++) {
-  queue = this.queues.get(i);
-  if (queue.size() != 0) {
-return queue;
+  private E removeNextElement() {
+int priority = multiplexer.getAndAdvanceCurrentIndex();
+E e = queues.get(priority).poll();
+if (e == null) {
+  for (int idx = 0; e == null && idx < queues.size(); idx++) {
+e = queues.get(idx).poll();
   }
 }
-
-// All queues were empty
-return null;
+// guaranteed to find an element if caller acquired permit.
+assert e != null : "consumer didn't acquire semaphore!";
+return e;
   }
 
   /* AbstractQueue and BlockingQueue methods */
@@ -184,9 +179,9 @@ public class FairCallQueue extends 
AbstractQueue
 int priorityLevel = e.getPriorityLevel();
 BlockingQueue q = this.queues.get(priorityLevel);
 boolean ret = q.offer(e, timeout, unit);
-
-signalNotEmpty();
-
+if (ret) {
+  signalNotEmpty();
+}
 return ret;
   }
 
@@ -195,72 +190,21 @@ public class FairCallQueue extends 
AbstractQueue
 int priorityLevel = e.getPriorityLevel();
 BlockingQueue q = this.queues.get(priorityLevel);
 boolean ret = q.offer(e);
-
-

hadoop git commit: HADOOP-14033. Reduce fair call queue lock contention. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 9b8505358 -> 0c01cf579


HADOOP-14033. Reduce fair call queue lock contention. Contributed by Daryn 
Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c01cf57
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c01cf57
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c01cf57

Branch: refs/heads/trunk
Commit: 0c01cf57987bcc7a17154a3538960b67f625a9e5
Parents: 9b85053
Author: Kihwal Lee 
Authored: Thu Feb 9 16:17:24 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 16:17:24 2017 -0600

--
 .../org/apache/hadoop/ipc/FairCallQueue.java| 167 ++-
 1 file changed, 51 insertions(+), 116 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c01cf57/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index 77a9d65..820f24c 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -27,8 +27,7 @@ import java.util.AbstractQueue;
 import java.util.HashMap;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.Condition;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -55,16 +54,15 @@ public class FairCallQueue extends 
AbstractQueue
   /* The queues */
   private final ArrayList queues;
 
-  /* Read locks */
-  private final ReentrantLock takeLock = new ReentrantLock();
-  private final Condition notEmpty = takeLock.newCondition();
+  /* Track available permits for scheduled objects.  All methods that will
+   * mutate a subqueue must acquire or release a permit on the semaphore.
+   * A semaphore is much faster than an exclusive lock because producers do
+   * not contend with consumers and consumers do not block other consumers
+   * while polling.
+   */
+  private final Semaphore semaphore = new Semaphore(0);
   private void signalNotEmpty() {
-takeLock.lock();
-try {
-  notEmpty.signal();
-} finally {
-  takeLock.unlock();
-}
+semaphore.release();
   }
 
   /* Multiplexer picks which queue to draw from */
@@ -112,28 +110,25 @@ public class FairCallQueue extends 
AbstractQueue
   }
 
   /**
-   * Returns the first non-empty queue with equal to startIdx, or
-   * or scans from highest to lowest priority queue.
+   * Returns an element first non-empty queue equal to the priority returned
+   * by the multiplexer or scans from highest to lowest priority queue.
+   *
+   * Caller must always acquire a semaphore permit before invoking.
*
-   * @param startIdx the queue number to start searching at
* @return the first non-empty queue with less priority, or null if
* everything was empty
*/
-  private BlockingQueue getFirstNonEmptyQueue(int startIdx) {
-BlockingQueue queue = this.queues.get(startIdx);
-if (queue.size() != 0) {
-  return queue;
-}
-final int numQueues = this.queues.size();
-for(int i=0; i < numQueues; i++) {
-  queue = this.queues.get(i);
-  if (queue.size() != 0) {
-return queue;
+  private E removeNextElement() {
+int priority = multiplexer.getAndAdvanceCurrentIndex();
+E e = queues.get(priority).poll();
+if (e == null) {
+  for (int idx = 0; e == null && idx < queues.size(); idx++) {
+e = queues.get(idx).poll();
   }
 }
-
-// All queues were empty
-return null;
+// guaranteed to find an element if caller acquired permit.
+assert e != null : "consumer didn't acquire semaphore!";
+return e;
   }
 
   /* AbstractQueue and BlockingQueue methods */
@@ -184,9 +179,9 @@ public class FairCallQueue extends 
AbstractQueue
 int priorityLevel = e.getPriorityLevel();
 BlockingQueue q = this.queues.get(priorityLevel);
 boolean ret = q.offer(e, timeout, unit);
-
-signalNotEmpty();
-
+if (ret) {
+  signalNotEmpty();
+}
 return ret;
   }
 
@@ -195,72 +190,21 @@ public class FairCallQueue extends 
AbstractQueue
 int priorityLevel = e.getPriorityLevel();
 BlockingQueue q = this.queues.get(priorityLevel);
 boolean ret = q.offer(e);
-
-signalNotEmpty();
-
+if (ret) {
+  signalNotEmpty();
+}
 return 

hadoop git commit: YARN-6112. UpdateCallDuration is calculated only when debug logging is enabled. (Yufei Gu via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d008b5515 -> 41361ec39


YARN-6112. UpdateCallDuration is calculated only when debug logging is enabled. 
(Yufei Gu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41361ec3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41361ec3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41361ec3

Branch: refs/heads/branch-2
Commit: 41361ec39e84ec9949285b86ed97888ed4871bc4
Parents: d008b55
Author: Karthik Kambatla 
Authored: Thu Feb 9 13:12:21 2017 -0800
Committer: Karthik Kambatla 
Committed: Thu Feb 9 13:12:21 2017 -0800

--
 .../server/resourcemanager/scheduler/fair/FairScheduler.java  | 7 +++
 1 file changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/41361ec3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 9694d62..f75f2dd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -312,6 +312,9 @@ public class FairScheduler extends
   long start = getClock().getTime();
   update();
   long duration = getClock().getTime() - start;
+  // UpdateCall duration and ThreadRun Duration are the same after
+  // YARN-4752 (preemption overhaul), we keep both for compatibility.
+  fsOpDurations.addUpdateCallDuration(duration);
   fsOpDurations.addUpdateThreadRunDuration(duration);
 } catch (InterruptedException ie) {
   LOG.warn("Update thread interrupted. Exiting.");
@@ -351,7 +354,6 @@ public class FairScheduler extends
   protected void update() {
 try {
   writeLock.lock();
-  long start = getClock().getTime();
 
   FSQueue rootQueue = queueMgr.getRootQueue();
 
@@ -374,9 +376,6 @@ public class FairScheduler extends
   rootMetrics.getAvailableVirtualCores()) +
   "  Demand: " + rootQueue.getDemand());
 }
-
-long duration = getClock().getTime() - start;
-fsOpDurations.addUpdateCallDuration(duration);
   }
 } finally {
   writeLock.unlock();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-6112. UpdateCallDuration is calculated only when debug logging is enabled. (Yufei Gu via kasha)

[kasha]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 5fb723bb7 -> 9b8505358


YARN-6112. UpdateCallDuration is calculated only when debug logging is enabled. 
(Yufei Gu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b850535
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b850535
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b850535

Branch: refs/heads/trunk
Commit: 9b85053583a3560f93062b656061d11b1b9c664f
Parents: 5fb723b
Author: Karthik Kambatla 
Authored: Thu Feb 9 13:09:32 2017 -0800
Committer: Karthik Kambatla 
Committed: Thu Feb 9 13:09:32 2017 -0800

--
 .../server/resourcemanager/scheduler/fair/FSOpDurations.java | 8 
 .../server/resourcemanager/scheduler/fair/FairScheduler.java | 4 
 2 files changed, 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b850535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
index 20d2af9..f6d843a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSOpDurations.java
@@ -50,9 +50,6 @@ public class FSOpDurations implements MetricsSource {
   @Metric("Duration for a update thread run")
   MutableRate updateThreadRun;
 
-  @Metric("Duration for an update call")
-  MutableRate updateCall;
-
   @Metric("Duration for a preempt call")
   MutableRate preemptCall;
 
@@ -87,7 +84,6 @@ public class FSOpDurations implements MetricsSource {
 continuousSchedulingRun.setExtended(isExtended);
 nodeUpdateCall.setExtended(isExtended);
 updateThreadRun.setExtended(isExtended);
-updateCall.setExtended(isExtended);
 preemptCall.setExtended(isExtended);
 
 INSTANCE.isExtended = isExtended;
@@ -110,10 +106,6 @@ public class FSOpDurations implements MetricsSource {
 updateThreadRun.add(value);
   }
 
-  public void addUpdateCallDuration(long value) {
-updateCall.add(value);
-  }
-
   public void addPreemptCallDuration(long value) {
 preemptCall.add(value);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b850535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 97871e7..134efff 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -350,7 +350,6 @@ public class FairScheduler extends
   protected void update() {
 try {
   writeLock.lock();
-  long start = getClock().getTime();
 
   FSQueue rootQueue = queueMgr.getRootQueue();
 
@@ -373,9 +372,6 @@ public class FairScheduler extends
   rootMetrics.getAvailableVirtualCores()) +
   "  Demand: " + rootQueue.getDemand());
 }
-
-long duration = getClock().getTime() - start;
-fsOpDurations.addUpdateCallDuration(duration);
   }
 } finally {
   writeLock.unlock();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[2/2] hadoop git commit: YARN-5889. Improve and refactor user-limit calculation in Capacity Scheduler. (Sunil G via wangda)

[wangda]

YARN-5889. Improve and refactor user-limit calculation in Capacity Scheduler. 
(Sunil G via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fb723bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fb723bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fb723bb

Branch: refs/heads/trunk
Commit: 5fb723bb77722d41df6959eee23e1b0cfeb5584e
Parents: b6bb99c
Author: Wangda Tan 
Authored: Thu Feb 9 10:23:50 2017 -0800
Committer: Wangda Tan 
Committed: Thu Feb 9 10:23:50 2017 -0800

--
 .../FifoIntraQueuePreemptionPlugin.java |   6 +-
 .../scheduler/AbstractUsersManager.java |  54 +
 .../scheduler/ActiveUsersManager.java   |  23 +-
 .../scheduler/AppSchedulingInfo.java|  16 +-
 .../server/resourcemanager/scheduler/Queue.java |   2 +-
 .../scheduler/SchedulerApplicationAttempt.java  |   4 +-
 .../scheduler/capacity/CSQueue.java |   8 +-
 .../capacity/CapacityHeadroomProvider.java  |   4 +-
 .../scheduler/capacity/CapacityScheduler.java   |   3 +-
 .../scheduler/capacity/LeafQueue.java   | 589 +++
 .../scheduler/capacity/ParentQueue.java |   2 +-
 .../scheduler/capacity/UsersManager.java| 982 +++
 .../scheduler/common/fica/FiCaSchedulerApp.java |  14 +-
 .../scheduler/fair/FSLeafQueue.java |   2 +-
 .../scheduler/fair/FSParentQueue.java   |   2 +-
 .../scheduler/fifo/FifoScheduler.java   |   2 +-
 .../dao/CapacitySchedulerLeafQueueInfo.java |   2 +-
 ...alCapacityPreemptionPolicyMockFramework.java |   8 +-
 .../TestSchedulerApplicationAttempt.java|  14 +-
 .../capacity/TestApplicationLimits.java |  15 +-
 .../TestApplicationLimitsByPartition.java   |  23 +-
 .../TestCapacitySchedulerNodeLabelUpdate.java   |   4 +-
 .../scheduler/capacity/TestLeafQueue.java   |  98 +-
 .../TestNodeLabelContainerAllocation.java   |   7 +-
 24 files changed, 1302 insertions(+), 582 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb723bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index 757f567..5f1af1e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
 import 
org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.IntraQueueCandidatesSelector.TAPriorityComparator;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -264,8 +265,9 @@ public class FifoIntraQueuePreemptionPlugin
 
   // Verify whether we already calculated headroom for this user.
   if (userLimitResource == null) {
-userLimitResource = Resources.clone(tq.leafQueue
-.getUserLimitPerUser(userName, partitionBasedResource, partition));
+userLimitResource = Resources.clone(
+tq.leafQueue.getResourceLimitForAllUsers(userName, clusterResource,
+partition, SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY));
 
 Resource amUsed = perUserAMUsed.get(userName);
 if (null == amUsed) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb723bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractUsersManager.java

[1/2] hadoop git commit: YARN-5889. Improve and refactor user-limit calculation in Capacity Scheduler. (Sunil G via wangda)

[wangda]

Repository: hadoop
Updated Branches:
  refs/heads/trunk b6bb99c18 -> 5fb723bb7


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fb723bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
new file mode 100644
index 000..05503c6
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -0,0 +1,982 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * {@link UsersManager} tracks users in the system and its respective data
+ * structures.
+ */
+@Private
+public class UsersManager implements AbstractUsersManager {
+
+  private static final Log LOG = LogFactory.getLog(UsersManager.class);
+
+  /*
+   * Member declaration for UsersManager class.
+   */
+  private final LeafQueue lQueue;
+  private final RMNodeLabelsManager labelManager;
+  private final ResourceCalculator resourceCalculator;
+  private final CapacitySchedulerContext scheduler;
+  private Map users = new ConcurrentHashMap<>();
+
+  private ResourceUsage totalResUsageForActiveUsers = new ResourceUsage();
+  private ResourceUsage totalResUsageForNonActiveUsers = new ResourceUsage();
+  private Set activeUsersSet = new HashSet();
+  private Set nonActiveUsersSet = new HashSet();
+
+  // Summation of consumed ratios for all users in queue
+  private UsageRatios qUsageRatios;
+
+  // To detect whether there is a change in user count for every user-limit
+  // calculation.
+  private AtomicLong latestVersionOfUsersState = new AtomicLong(0);
+  private Map> 
localVersionOfActiveUsersState =
+  new HashMap>();
+  private Map> localVersionOfAllUsersState =
+  new HashMap>();
+
+  private volatile int userLimit;
+  private volatile float userLimitFactor;
+
+  private WriteLock writeLock;
+  private ReadLock readLock;
+
+  private final QueueMetrics metrics;
+  private AtomicInteger activeUsers = new AtomicInteger(0);
+  private Map usersApplications =
+  new HashMap();
+
+  // Pre-computed list of user-limits.
+  Map> 

hadoop git commit: HADOOP-14034. Allow ipc layer exceptions to selectively close connections. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 bb98856af -> 47bbe431b


HADOOP-14034. Allow ipc layer exceptions to selectively close connections. 
Contributed by Daryn Sharp.

(cherry picked from commit d008b5515304b42faeb48e542c8c27586b8564eb)

Conflicts:

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47bbe431
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47bbe431
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47bbe431

Branch: refs/heads/branch-2.8
Commit: 47bbe431bf85727357feb85650c1b57a3e99f113
Parents: bb98856
Author: Kihwal Lee 
Authored: Thu Feb 9 11:04:29 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 11:04:29 2017 -0600

--
 .../main/java/org/apache/hadoop/ipc/Server.java | 213 ++-
 .../java/org/apache/hadoop/ipc/TestRPC.java | 119 +++
 2 files changed, 235 insertions(+), 97 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47bbe431/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index c99d553..ccdd776 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -1127,20 +1127,16 @@ public abstract class Server {
 LOG.info(Thread.currentThread().getName() + ": readAndProcess caught 
InterruptedException", ieo);
 throw ieo;
   } catch (Exception e) {
-// Do not log WrappedRpcServerExceptionSuppressed.
-if (!(e instanceof WrappedRpcServerExceptionSuppressed)) {
-  // A WrappedRpcServerException is an exception that has been sent
-  // to the client, so the stacktrace is unnecessary; any other
-  // exceptions are unexpected internal server errors and thus the
-  // stacktrace should be logged.
-  LOG.info(Thread.currentThread().getName() +
-  ": readAndProcess from client " + c.getHostAddress() +
-  " threw exception [" + e + "]",
-  (e instanceof WrappedRpcServerException) ? null : e);
-}
+// Any exceptions that reach here are fatal unexpected internal errors
+// that could not be sent to the client.
+LOG.info(Thread.currentThread().getName() +
+": readAndProcess from client " + c +
+" threw exception [" + e + "]", e);
 count = -1; //so that the (count < 0) block is executed
   }
-  if (count < 0) {
+  // setupResponse will signal the connection should be closed when a
+  // fatal response is sent.
+  if (count < 0 || c.shouldClose()) {
 closeConnection(c);
 c = null;
   }
@@ -1468,16 +1464,20 @@ public abstract class Server {
* unnecessary stack trace logging if it's not an internal server error. 
*/
   @SuppressWarnings("serial")
-  private static class WrappedRpcServerException extends RpcServerException {
+  private static class FatalRpcServerException extends RpcServerException {
 private final RpcErrorCodeProto errCode;
-public WrappedRpcServerException(RpcErrorCodeProto errCode, IOException 
ioe) {
+public FatalRpcServerException(RpcErrorCodeProto errCode, IOException ioe) 
{
   super(ioe.toString(), ioe);
   this.errCode = errCode;
 }
-public WrappedRpcServerException(RpcErrorCodeProto errCode, String 
message) {
+public FatalRpcServerException(RpcErrorCodeProto errCode, String message) {
   this(errCode, new RpcServerException(message));
 }
 @Override
+public RpcStatusProto getRpcStatusProto() {
+  return RpcStatusProto.FATAL;
+}
+@Override
 public RpcErrorCodeProto getRpcErrorCodeProto() {
   return errCode;
 }
@@ -1487,19 +1487,6 @@ public abstract class Server {
 }
   }
 
-  /**
-   * A WrappedRpcServerException that is suppressed altogether
-   * for the purposes of logging.
-   */
-  @SuppressWarnings("serial")
-  private static class WrappedRpcServerExceptionSuppressed
-  extends WrappedRpcServerException {
-public WrappedRpcServerExceptionSuppressed(
-RpcErrorCodeProto errCode, IOException ioe) {
-  super(errCode, ioe);
-}
-  }
-
   /** Reads calls from a connection and queues them for handling. */
   public class Connection {
 private boolean connectionHeaderRead = false; // connection  header is 
read?
@@ -1531,7 

hadoop git commit: HADOOP-14034. Allow ipc layer exceptions to selectively close connections. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a0bfb4150 -> b6bb99c18


HADOOP-14034. Allow ipc layer exceptions to selectively close connections. 
Contributed by Daryn Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6bb99c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6bb99c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6bb99c1

Branch: refs/heads/trunk
Commit: b6bb99c18a772d2179d5cc6757cddf141e8d39c0
Parents: a0bfb41
Author: Kihwal Lee 
Authored: Thu Feb 9 10:47:29 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 10:47:29 2017 -0600

--
 .../main/java/org/apache/hadoop/ipc/Server.java | 206 ++-
 .../java/org/apache/hadoop/ipc/TestRPC.java | 119 +++
 2 files changed, 227 insertions(+), 98 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6bb99c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 5ed6e15..3ca1524 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -1241,20 +1241,16 @@ public abstract class Server {
 LOG.info(Thread.currentThread().getName() + ": readAndProcess caught 
InterruptedException", ieo);
 throw ieo;
   } catch (Exception e) {
-// Do not log WrappedRpcServerExceptionSuppressed.
-if (!(e instanceof WrappedRpcServerExceptionSuppressed)) {
-  // A WrappedRpcServerException is an exception that has been sent
-  // to the client, so the stacktrace is unnecessary; any other
-  // exceptions are unexpected internal server errors and thus the
-  // stacktrace should be logged.
-  LOG.info(Thread.currentThread().getName() +
-  ": readAndProcess from client " + c.getHostAddress() +
-  " threw exception [" + e + "]",
-  (e instanceof WrappedRpcServerException) ? null : e);
-}
+// Any exceptions that reach here are fatal unexpected internal errors
+// that could not be sent to the client.
+LOG.info(Thread.currentThread().getName() +
+": readAndProcess from client " + c +
+" threw exception [" + e + "]", e);
 count = -1; //so that the (count < 0) block is executed
   }
-  if (count < 0) {
+  // setupResponse will signal the connection should be closed when a
+  // fatal response is sent.
+  if (count < 0 || c.shouldClose()) {
 closeConnection(c);
 c = null;
   }
@@ -1582,16 +1578,20 @@ public abstract class Server {
* unnecessary stack trace logging if it's not an internal server error. 
*/
   @SuppressWarnings("serial")
-  private static class WrappedRpcServerException extends RpcServerException {
+  private static class FatalRpcServerException extends RpcServerException {
 private final RpcErrorCodeProto errCode;
-public WrappedRpcServerException(RpcErrorCodeProto errCode, IOException 
ioe) {
+public FatalRpcServerException(RpcErrorCodeProto errCode, IOException ioe) 
{
   super(ioe.toString(), ioe);
   this.errCode = errCode;
 }
-public WrappedRpcServerException(RpcErrorCodeProto errCode, String 
message) {
+public FatalRpcServerException(RpcErrorCodeProto errCode, String message) {
   this(errCode, new RpcServerException(message));
 }
 @Override
+public RpcStatusProto getRpcStatusProto() {
+  return RpcStatusProto.FATAL;
+}
+@Override
 public RpcErrorCodeProto getRpcErrorCodeProto() {
   return errCode;
 }
@@ -1601,19 +1601,6 @@ public abstract class Server {
 }
   }
 
-  /**
-   * A WrappedRpcServerException that is suppressed altogether
-   * for the purposes of logging.
-   */
-  @SuppressWarnings("serial")
-  private static class WrappedRpcServerExceptionSuppressed
-  extends WrappedRpcServerException {
-public WrappedRpcServerExceptionSuppressed(
-RpcErrorCodeProto errCode, IOException ioe) {
-  super(errCode, ioe);
-}
-  }
-
   /** Reads calls from a connection and queues them for handling. */
   public class Connection {
 private boolean connectionHeaderRead = false; // connection  header is 
read?
@@ -1645,7 +1632,8 @@ public abstract class Server {
 private ByteBuffer unwrappedData;
 private ByteBuffer unwrappedDataLengthBuffer;
 private int serviceClass;
-
+private 

hadoop git commit: HADOOP-14034. Allow ipc layer exceptions to selectively close connections. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2039d021f -> d008b5515


HADOOP-14034. Allow ipc layer exceptions to selectively close connections. 
Contributed by Daryn Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d008b551
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d008b551
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d008b551

Branch: refs/heads/branch-2
Commit: d008b5515304b42faeb48e542c8c27586b8564eb
Parents: 2039d02
Author: Kihwal Lee 
Authored: Thu Feb 9 10:55:40 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 10:55:40 2017 -0600

--
 .../main/java/org/apache/hadoop/ipc/Server.java | 209 ++-
 .../java/org/apache/hadoop/ipc/TestRPC.java | 119 +++
 2 files changed, 225 insertions(+), 103 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d008b551/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index dc6661b..890569c 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -1240,20 +1240,16 @@ public abstract class Server {
 LOG.info(Thread.currentThread().getName() + ": readAndProcess caught 
InterruptedException", ieo);
 throw ieo;
   } catch (Exception e) {
-// Do not log WrappedRpcServerExceptionSuppressed.
-if (!(e instanceof WrappedRpcServerExceptionSuppressed)) {
-  // A WrappedRpcServerException is an exception that has been sent
-  // to the client, so the stacktrace is unnecessary; any other
-  // exceptions are unexpected internal server errors and thus the
-  // stacktrace should be logged.
-  LOG.info(Thread.currentThread().getName() +
-  ": readAndProcess from client " + c.getHostAddress() +
-  " threw exception [" + e + "]",
-  (e instanceof WrappedRpcServerException) ? null : e);
-}
+// Any exceptions that reach here are fatal unexpected internal errors
+// that could not be sent to the client.
+LOG.info(Thread.currentThread().getName() +
+": readAndProcess from client " + c +
+" threw exception [" + e + "]", e);
 count = -1; //so that the (count < 0) block is executed
   }
-  if (count < 0) {
+  // setupResponse will signal the connection should be closed when a
+  // fatal response is sent.
+  if (count < 0 || c.shouldClose()) {
 closeConnection(c);
 c = null;
   }
@@ -1581,16 +1577,20 @@ public abstract class Server {
* unnecessary stack trace logging if it's not an internal server error. 
*/
   @SuppressWarnings("serial")
-  private static class WrappedRpcServerException extends RpcServerException {
+  private static class FatalRpcServerException extends RpcServerException {
 private final RpcErrorCodeProto errCode;
-public WrappedRpcServerException(RpcErrorCodeProto errCode, IOException 
ioe) {
+public FatalRpcServerException(RpcErrorCodeProto errCode, IOException ioe) 
{
   super(ioe.toString(), ioe);
   this.errCode = errCode;
 }
-public WrappedRpcServerException(RpcErrorCodeProto errCode, String 
message) {
+public FatalRpcServerException(RpcErrorCodeProto errCode, String message) {
   this(errCode, new RpcServerException(message));
 }
 @Override
+public RpcStatusProto getRpcStatusProto() {
+  return RpcStatusProto.FATAL;
+}
+@Override
 public RpcErrorCodeProto getRpcErrorCodeProto() {
   return errCode;
 }
@@ -1600,19 +1600,6 @@ public abstract class Server {
 }
   }
 
-  /**
-   * A WrappedRpcServerException that is suppressed altogether
-   * for the purposes of logging.
-   */
-  @SuppressWarnings("serial")
-  private static class WrappedRpcServerExceptionSuppressed
-  extends WrappedRpcServerException {
-public WrappedRpcServerExceptionSuppressed(
-RpcErrorCodeProto errCode, IOException ioe) {
-  super(errCode, ioe);
-}
-  }
-
   /** Reads calls from a connection and queues them for handling. */
   public class Connection {
 private boolean connectionHeaderRead = false; // connection  header is 
read?
@@ -1644,7 +1631,8 @@ public abstract class Server {
 private ByteBuffer unwrappedData;
 private ByteBuffer unwrappedDataLengthBuffer;
 private int serviceClass;
-
+private 

hadoop git commit: HADOOP-14032. Reduce fair call queue priority inversion. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 04076e783 -> bb98856af


HADOOP-14032. Reduce fair call queue priority inversion. Contributed by Daryn 
Sharp.

(cherry picked from commit a0bfb4150464013a618f30c2e38d88fc6de11ad2)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb98856a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb98856a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb98856a

Branch: refs/heads/branch-2.8
Commit: bb98856af913801909c5c5a80f87e50dd73967d1
Parents: 04076e7
Author: Kihwal Lee 
Authored: Thu Feb 9 10:14:18 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 10:14:18 2017 -0600

--
 .../org/apache/hadoop/ipc/FairCallQueue.java| 12 +++--
 .../apache/hadoop/ipc/TestFairCallQueue.java| 57 +++-
 2 files changed, 63 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb98856a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index c2d3cd8..77a9d65 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -112,19 +112,21 @@ public class FairCallQueue extends 
AbstractQueue
   }
 
   /**
-   * Returns the first non-empty queue with equal or lesser priority
-   * than startIdx. Wraps around, searching a maximum of N
-   * queues, where N is this.queues.size().
+   * Returns the first non-empty queue with equal to startIdx, or
+   * or scans from highest to lowest priority queue.
*
* @param startIdx the queue number to start searching at
* @return the first non-empty queue with less priority, or null if
* everything was empty
*/
   private BlockingQueue getFirstNonEmptyQueue(int startIdx) {
+BlockingQueue queue = this.queues.get(startIdx);
+if (queue.size() != 0) {
+  return queue;
+}
 final int numQueues = this.queues.size();
 for(int i=0; i < numQueues; i++) {
-  int idx = (i + startIdx) % numQueues; // offset and wrap around
-  BlockingQueue queue = this.queues.get(idx);
+  queue = this.queues.get(i);
   if (queue.size() != 0) {
 return queue;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb98856a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index 96dea80..901a771 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -28,9 +28,12 @@ import javax.management.ObjectName;
 import java.lang.management.ManagementFactory;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.BlockingQueue;
-
 import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
 import org.apache.hadoop.conf.Configuration;
 
 public class TestFairCallQueue extends TestCase {
@@ -43,6 +46,7 @@ public class TestFairCallQueue extends TestCase {
 when(ugi.getUserName()).thenReturn(id);
 when(mockCall.getUserGroupInformation()).thenReturn(ugi);
 when(mockCall.getPriorityLevel()).thenReturn(priority);
+when(mockCall.toString()).thenReturn("id=" + id + " priority=" + priority);
 
 return mockCall;
   }
@@ -78,6 +82,57 @@ public class TestFairCallQueue extends TestCase {
 assertEquals(fairCallQueue.remainingCapacity(), 1025);
   }
 
+  @Test
+  public void testPrioritization() {
+int numQueues = 10;
+Configuration conf = new Configuration();
+fcq = new FairCallQueue(numQueues, numQueues, "ns", conf);
+
+//Schedulable[] calls = new Schedulable[numCalls];
+List calls = new ArrayList<>();
+for (int i=0; i < numQueues; i++) {
+  Schedulable call = mockCall("u", i);
+  calls.add(call);
+  fcq.add(call);
+}
+
+final AtomicInteger currentIndex = new AtomicInteger();
+fcq.setMultiplexer(new RpcMultiplexer(){
+  @Override
+  

hadoop git commit: HADOOP-14032. Reduce fair call queue priority inversion. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e330c3806 -> 2039d021f


HADOOP-14032. Reduce fair call queue priority inversion. Contributed by Daryn 
Sharp.

(cherry picked from commit a0bfb4150464013a618f30c2e38d88fc6de11ad2)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2039d021
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2039d021
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2039d021

Branch: refs/heads/branch-2
Commit: 2039d021f986ef09b0c132c2a35463f8cbd50331
Parents: e330c38
Author: Kihwal Lee 
Authored: Thu Feb 9 10:13:37 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 10:13:37 2017 -0600

--
 .../org/apache/hadoop/ipc/FairCallQueue.java| 12 +++--
 .../apache/hadoop/ipc/TestFairCallQueue.java| 57 +++-
 2 files changed, 63 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2039d021/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index c2d3cd8..77a9d65 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -112,19 +112,21 @@ public class FairCallQueue extends 
AbstractQueue
   }
 
   /**
-   * Returns the first non-empty queue with equal or lesser priority
-   * than startIdx. Wraps around, searching a maximum of N
-   * queues, where N is this.queues.size().
+   * Returns the first non-empty queue with equal to startIdx, or
+   * or scans from highest to lowest priority queue.
*
* @param startIdx the queue number to start searching at
* @return the first non-empty queue with less priority, or null if
* everything was empty
*/
   private BlockingQueue getFirstNonEmptyQueue(int startIdx) {
+BlockingQueue queue = this.queues.get(startIdx);
+if (queue.size() != 0) {
+  return queue;
+}
 final int numQueues = this.queues.size();
 for(int i=0; i < numQueues; i++) {
-  int idx = (i + startIdx) % numQueues; // offset and wrap around
-  BlockingQueue queue = this.queues.get(idx);
+  queue = this.queues.get(i);
   if (queue.size() != 0) {
 return queue;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2039d021/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index 96dea80..901a771 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -28,9 +28,12 @@ import javax.management.ObjectName;
 import java.lang.management.ManagementFactory;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.BlockingQueue;
-
 import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
 import org.apache.hadoop.conf.Configuration;
 
 public class TestFairCallQueue extends TestCase {
@@ -43,6 +46,7 @@ public class TestFairCallQueue extends TestCase {
 when(ugi.getUserName()).thenReturn(id);
 when(mockCall.getUserGroupInformation()).thenReturn(ugi);
 when(mockCall.getPriorityLevel()).thenReturn(priority);
+when(mockCall.toString()).thenReturn("id=" + id + " priority=" + priority);
 
 return mockCall;
   }
@@ -78,6 +82,57 @@ public class TestFairCallQueue extends TestCase {
 assertEquals(fairCallQueue.remainingCapacity(), 1025);
   }
 
+  @Test
+  public void testPrioritization() {
+int numQueues = 10;
+Configuration conf = new Configuration();
+fcq = new FairCallQueue(numQueues, numQueues, "ns", conf);
+
+//Schedulable[] calls = new Schedulable[numCalls];
+List calls = new ArrayList<>();
+for (int i=0; i < numQueues; i++) {
+  Schedulable call = mockCall("u", i);
+  calls.add(call);
+  fcq.add(call);
+}
+
+final AtomicInteger currentIndex = new AtomicInteger();
+fcq.setMultiplexer(new RpcMultiplexer(){
+  @Override
+  

hadoop git commit: HADOOP-14032. Reduce fair call queue priority inversion. Contributed by Daryn Sharp.

[kihwal]

Repository: hadoop
Updated Branches:
  refs/heads/trunk a8a594b4c -> a0bfb4150


HADOOP-14032. Reduce fair call queue priority inversion. Contributed by Daryn 
Sharp.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0bfb415
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0bfb415
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0bfb415

Branch: refs/heads/trunk
Commit: a0bfb4150464013a618f30c2e38d88fc6de11ad2
Parents: a8a594b
Author: Kihwal Lee 
Authored: Thu Feb 9 10:04:28 2017 -0600
Committer: Kihwal Lee 
Committed: Thu Feb 9 10:04:28 2017 -0600

--
 .../org/apache/hadoop/ipc/FairCallQueue.java| 12 +++--
 .../apache/hadoop/ipc/TestFairCallQueue.java| 57 +++-
 2 files changed, 63 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0bfb415/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
index c2d3cd8..77a9d65 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/FairCallQueue.java
@@ -112,19 +112,21 @@ public class FairCallQueue extends 
AbstractQueue
   }
 
   /**
-   * Returns the first non-empty queue with equal or lesser priority
-   * than startIdx. Wraps around, searching a maximum of N
-   * queues, where N is this.queues.size().
+   * Returns the first non-empty queue with equal to startIdx, or
+   * or scans from highest to lowest priority queue.
*
* @param startIdx the queue number to start searching at
* @return the first non-empty queue with less priority, or null if
* everything was empty
*/
   private BlockingQueue getFirstNonEmptyQueue(int startIdx) {
+BlockingQueue queue = this.queues.get(startIdx);
+if (queue.size() != 0) {
+  return queue;
+}
 final int numQueues = this.queues.size();
 for(int i=0; i < numQueues; i++) {
-  int idx = (i + startIdx) % numQueues; // offset and wrap around
-  BlockingQueue queue = this.queues.get(idx);
+  queue = this.queues.get(i);
   if (queue.size() != 0) {
 return queue;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0bfb415/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
index 96dea80..901a771 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestFairCallQueue.java
@@ -28,9 +28,12 @@ import javax.management.ObjectName;
 import java.lang.management.ManagementFactory;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.BlockingQueue;
-
 import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
 import org.apache.hadoop.conf.Configuration;
 
 public class TestFairCallQueue extends TestCase {
@@ -43,6 +46,7 @@ public class TestFairCallQueue extends TestCase {
 when(ugi.getUserName()).thenReturn(id);
 when(mockCall.getUserGroupInformation()).thenReturn(ugi);
 when(mockCall.getPriorityLevel()).thenReturn(priority);
+when(mockCall.toString()).thenReturn("id=" + id + " priority=" + priority);
 
 return mockCall;
   }
@@ -78,6 +82,57 @@ public class TestFairCallQueue extends TestCase {
 assertEquals(fairCallQueue.remainingCapacity(), 1025);
   }
 
+  @Test
+  public void testPrioritization() {
+int numQueues = 10;
+Configuration conf = new Configuration();
+fcq = new FairCallQueue(numQueues, numQueues, "ns", conf);
+
+//Schedulable[] calls = new Schedulable[numCalls];
+List calls = new ArrayList<>();
+for (int i=0; i < numQueues; i++) {
+  Schedulable call = mockCall("u", i);
+  calls.add(call);
+  fcq.add(call);
+}
+
+final AtomicInteger currentIndex = new AtomicInteger();
+fcq.setMultiplexer(new RpcMultiplexer(){
+  @Override
+  public int getAndAdvanceCurrentIndex() {
+return currentIndex.get();
+  

hadoop git commit: HADOOP-14065. AliyunOSS: oss directory filestatus should use meta time. Contributed by Fei Hui

[drankye]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 37b4acf7c -> a8a594b4c


HADOOP-14065. AliyunOSS: oss directory filestatus should use meta time. 
Contributed by Fei Hui


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8a594b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8a594b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8a594b4

Branch: refs/heads/trunk
Commit: a8a594b4c89319bef294534755f0e4ed6198ec88
Parents: 37b4acf
Author: Kai Zheng 
Authored: Thu Feb 9 18:00:22 2017 +0800
Committer: Kai Zheng 
Committed: Thu Feb 9 18:00:22 2017 +0800

--
 .../apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java| 3 ++-
 .../hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java  | 9 +
 2 files changed, 11 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8a594b4/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 3b266c8..3426319 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -218,7 +218,8 @@ public class AliyunOSSFileSystem extends FileSystem {
 throw new FileNotFoundException(path + ": No such file or directory!");
   }
 } else if (objectRepresentsDirectory(key, meta.getContentLength())) {
-  return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+  return new FileStatus(0, true, 1, 0, meta.getLastModified().getTime(),
+   qualifiedPath);
 } else {
   return new FileStatus(meta.getContentLength(), false, 1,
   getDefaultBlockSize(path), meta.getLastModified().getTime(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8a594b4/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
 
b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
index 37af28f..10c4edd 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.fs.FileStatus;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
@@ -142,4 +143,12 @@ public class TestAliyunOSSInputStream {
 assertTrue(instream.available() == 0);
 IOUtils.closeStream(instream);
   }
+  @Test
+  public void testDirectoryModifiedTime() throws Exception {
+Path emptyDirPath = setPath("/test/emptyDirectory");
+fs.mkdirs(emptyDirPath);
+FileStatus dirFileStatus = fs.getFileStatus(emptyDirPath);
+assertTrue("expected the empty dir is new",
+dirFileStatus.getModificationTime() > 0L);
+  }
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org