hadoop git commit: YARN-7863. Modify placement constraints to support node attributes. Contributed by Sunil Govindan.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 0f0b55f7f -> 87f236cf0


YARN-7863. Modify placement constraints to support node attributes. Contributed 
by Sunil Govindan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/87f236cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/87f236cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/87f236cf

Branch: refs/heads/YARN-3409
Commit: 87f236cf000241d47928c4716868012e675285af
Parents: 0f0b55f
Author: Naganarasimha 
Authored: Mon Aug 27 10:27:33 2018 +0800
Committer: Naganarasimha 
Committed: Mon Aug 27 10:27:33 2018 +0800

--
 .../yarn/api/records/NodeAttributeOpCode.java   |  43 +++
 .../yarn/api/resource/PlacementConstraint.java  |  40 +-
 .../yarn/api/resource/PlacementConstraints.java |  19 +++
 .../constraint/PlacementConstraintParser.java   | 112 +++--
 .../src/main/proto/yarn_protos.proto|   7 ++
 .../resource/TestPlacementConstraintParser.java |  61 -
 .../distributedshell/ApplicationMaster.java |  35 --
 .../applications/distributedshell/Client.java   |   5 +-
 .../distributedshell/PlacementSpec.java |  19 ++-
 .../PlacementConstraintFromProtoConverter.java  |  10 +-
 .../pb/PlacementConstraintToProtoConverter.java |  11 ++
 .../server/resourcemanager/ResourceManager.java |   7 +-
 .../nodelabels/NodeAttributesManagerImpl.java   |  24 +++-
 .../scheduler/SchedulerNode.java|  11 ++
 .../scheduler/capacity/CapacityScheduler.java   |  36 +-
 .../constraint/PlacementConstraintsUtil.java| 126 ---
 .../NodeAttributesUpdateSchedulerEvent.java |  41 ++
 .../scheduler/event/SchedulerEventType.java |   1 +
 .../LocalityAppPlacementAllocator.java  |   4 +
 19 files changed, 567 insertions(+), 45 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f236cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java
new file mode 100644
index 000..76db063
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeOpCode.java
@@ -0,0 +1,43 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+/**
+ * Enumeration of various node attribute op codes.
+ */
+@Public
+@Evolving
+public enum NodeAttributeOpCode {
+  /**
+   * Default as No OP.
+   */
+  NO_OP,
+  /**
+   * EQUALS op code for Attribute.
+   */
+  EQ,
+
+  /**
+   * NOT EQUALS op code for Attribute.
+   */
+  NE
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/87f236cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index 0fe8273..79196fb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -29,6 +29,7 @@ import java.util.Iterator;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import 

hadoop git commit: YARN-8574. Allow dot in attribute values. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 acd7729af -> 7ad66e806


YARN-8574. Allow dot in attribute values. Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ad66e80
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ad66e80
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ad66e80

Branch: refs/heads/YARN-3409
Commit: 7ad66e8065fc941136c47a5118d3d5c835a17ef5
Parents: acd7729
Author: Naganarasimha 
Authored: Sun Jul 29 13:52:52 2018 +0800
Committer: Naganarasimha 
Committed: Sun Jul 29 13:54:08 2018 +0800

--
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   |  4 +-
 .../yarn/nodelabels/TestNodeLabelUtil.java  | 47 
 2 files changed, 50 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad66e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
index 0dd0755..395ff81 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -38,6 +38,8 @@ public final class NodeLabelUtil {
   Pattern.compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
   private static final Pattern PREFIX_PATTERN =
   Pattern.compile("^[0-9a-zA-Z][0-9a-zA-Z-_\\.]*");
+  private static final Pattern ATTRIBUTE_VALUE_PATTERN =
+  Pattern.compile("^[0-9a-zA-Z][0-9a-zA-Z-_.]*");
 
   public static void checkAndThrowLabelName(String label) throws IOException {
 if (label == null || label.isEmpty() || label.length() > MAX_LABEL_LENGTH) 
{
@@ -69,7 +71,7 @@ public final class NodeLabelUtil {
   return;
 }
 
-boolean match = LABEL_OR_VALUE_PATTERN.matcher(value).matches();
+boolean match = ATTRIBUTE_VALUE_PATTERN.matcher(value).matches();
 
 if (!match) {
   throw new IOException("attribute value should only contains "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ad66e80/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
new file mode 100644
index 000..d43da4f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestNodeLabelUtil.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels;
+
+import static org.junit.Assert.fail;
+import org.junit.Test;
+
+public class TestNodeLabelUtil {
+
+  @Test
+  public void testAttributeValueAddition() {
+String[] values =
+new String[] {"1_8", "1.8", "ABZ", "ABZ", "az", "a-z","a_z", 
"123456789"};
+for (String val : values) {
+  try {
+NodeLabelUtil.checkAndThrowAttributeValue(val);
+  } catch (Exception e) {
+fail("Valid values for NodeAttributeValue :" + val);
+  }
+}
+
+String[] invalidVals = new String[] {"_18","1,8","1/5",".15","1\\5"};
+for (String val : invalidVals) {
+  try {
+NodeLabelUtil.checkAndThrowAttributeValue(val);
+fail("Valid values for NodeAttributeValue :" + val);
+  } catch (Exception e) {
+// IGNORE
+  }
+}
+  }
+}

[1/2] hadoop git commit: YARN-8103. Add CLI interface to query node attributes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 4ba4ebb9e -> 81e410404


http://git-wip-us.apache.org/repos/asf/hadoop/blob/81e41040/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 0002567..689e2e6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
@@ -152,15 +152,19 @@ public class NodeAttributePBImpl extends NodeAttribute {
 }
 if (obj instanceof NodeAttribute) {
   NodeAttribute other = (NodeAttribute) obj;
-  getAttributeKey().equals(other.getAttributeKey());
-  return true;
+  return getAttributeKey().equals(other.getAttributeKey());
 }
 return false;
   }
 
   @Override
   public String toString() {
-return getAttributeKey().toString() + ":Value-" + getAttributeValue()
-+ ":Type-" + getAttributeType();
+StringBuilder strBuilder = new StringBuilder();
+NodeAttributeKey key = this.getAttributeKey();
+strBuilder.append(key.getAttributePrefix()).append("/")
+.append(key.getAttributeName()).append("(")
+.append(this.getAttributeType()).append(")=")
+.append(this.getAttributeValue());
+return strBuilder.toString();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81e41040/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index ced588d..7d5a06a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -18,17 +18,21 @@
 
 package org.apache.hadoop.yarn.api.records.impl.pb;
 
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeReport;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.NodeUpdateType;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
 import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProtoOrBuilder;
@@ -50,6 +54,7 @@ public class NodeReportPBImpl extends NodeReport {
   private ResourceUtilization containersUtilization = null;
   private ResourceUtilization nodeUtilization = null;
   Set labels;
+  private Set nodeAttributes;
 
   public NodeReportPBImpl() {
 builder = NodeReportProto.newBuilder();
@@ -268,6 +273,14 @@ public class NodeReportPBImpl extends NodeReport {
   builder.clearNodeLabels();
   builder.addAllNodeLabels(this.labels);
 }
+if (this.nodeAttributes != null) {
+  builder.clearNodeAttributes();
+  List attrList = new ArrayList<>();
+  for (NodeAttribute attr : this.nodeAttributes) {
+attrList.add(convertToProtoFormat(attr));
+  }
+  builder.addAllNodeAttributes(attrList);
+}
 if (this.nodeUtilization != null
 && !((ResourceUtilizationPBImpl) this.nodeUtilization).getProto()
 .equals(builder.getNodeUtilization())) {
@@ -306,7 +319,16 @@ public class NodeReportPBImpl extends NodeReport {
   private NodeIdProto convertToProtoFormat(NodeId nodeId) {
 return ((NodeIdPBImpl) nodeId).getProto();
   }
-  
+
+  private NodeAttributeProto convertToProtoFormat(NodeAttribute nodeAttr) {
+return ((NodeAttributePBImpl) nodeAttr).getProto();
+  }
+
+  private NodeAttributePBImpl convertFromProtoFormat(
+  NodeAttributeProto nodeAttr) {
+

[2/2] hadoop git commit: YARN-8103. Add CLI interface to query node attributes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-8103. Add CLI interface to query node attributes. Contributed by Bibin A 
Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81e41040
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81e41040
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81e41040

Branch: refs/heads/YARN-3409
Commit: 81e41040467a7641dd321c2efd6a96a87b1e1597
Parents: 4ba4ebb
Author: Naganarasimha 
Authored: Thu Jun 28 08:13:09 2018 +0800
Committer: Naganarasimha 
Committed: Thu Jun 28 08:13:09 2018 +0800

--
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   |  12 +-
 .../yarn/sls/scheduler/RMNodeWrapper.java   |   8 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   4 +-
 .../hadoop/yarn/api/records/NodeReport.java |  13 +
 .../src/main/proto/yarn_protos.proto|   1 +
 .../hadoop/yarn/client/cli/ClusterCLI.java  |  17 +
 .../yarn/client/cli/NodeAttributesCLI.java  | 893 +--
 .../apache/hadoop/yarn/client/cli/NodeCLI.java  |  13 +-
 .../hadoop/yarn/client/cli/TestClusterCLI.java  |  32 +-
 .../yarn/client/cli/TestNodeAttributesCLI.java  | 331 +--
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  31 +-
 .../impl/pb/NodeAttributeInfoPBImpl.java|  10 +-
 .../records/impl/pb/NodeAttributePBImpl.java|  12 +-
 .../api/records/impl/pb/NodeReportPBImpl.java   |  44 +-
 .../hadoop/yarn/server/utils/BuilderUtils.java  |   6 +-
 .../server/resourcemanager/AdminService.java|   5 +-
 .../server/resourcemanager/ClientRMService.java |   5 +-
 .../resourcemanager/ResourceTrackerService.java |   4 -
 .../server/resourcemanager/rmnode/RMNode.java   |  11 +-
 .../resourcemanager/rmnode/RMNodeImpl.java  |  18 +-
 .../resourcemanager/webapp/dao/NodeInfo.java|  13 +-
 .../yarn/server/resourcemanager/MockNodes.java  |   4 +-
 .../resourcemanager/TestRMAdminService.java |  14 +-
 23 files changed, 1049 insertions(+), 452 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/81e41040/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 29d54a1..9c3d345 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.sls.nodemanager;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -219,16 +220,9 @@ public class NodeInfo {
   return null;
 }
 
-
-@Override
-public void setNodeAttributes(String prefix,
-Set nodeAttributes) {
-
-}
-
 @Override
-public Map> getAllNodeAttributes() {
-  return null;
+public Set getAllNodeAttributes() {
+  return Collections.emptySet();
 }
 
 @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81e41040/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index a603036..318d4be 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -208,13 +208,7 @@ public class RMNodeWrapper implements RMNode {
   }
   
   @Override
-  public void setNodeAttributes(String prefix,
-  Set nodeAttributes) {
-node.setNodeAttributes(prefix, nodeAttributes);
-  }
-
-  @Override
-  public Map> getAllNodeAttributes() {
+  public Set getAllNodeAttributes() {
 return node.getAllNodeAttributes();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81e41040/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 7cd838f..8290fcd 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -55,7 +55,7 @@ function hadoop_usage
   hadoop_add_subcommand "timelinereader" client "run the timeline reader 
server"
   hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
   hadoop_add_subcommand "top" 

hadoop git commit: YARN-8288. Fix wrong number of table columns in Resource Model doc. Contributed by Weiwei Yang.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 a700d05f7 -> ee7b2fd5d


YARN-8288. Fix wrong number of table columns in Resource Model doc. Contributed 
by Weiwei Yang.

(cherry picked from commit 8a2b5914f3a68148f40f99105acf5dafcc326e89)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee7b2fd5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee7b2fd5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee7b2fd5

Branch: refs/heads/branch-3.0
Commit: ee7b2fd5de2abc17648f6ee1a55637d9b2a4f5e0
Parents: a700d05
Author: Naganarasimha 
Authored: Tue May 15 00:03:38 2018 +0800
Committer: Naganarasimha 
Committed: Tue May 15 00:12:45 2018 +0800

--
 .../hadoop-yarn-site/src/site/markdown/ResourceModel.md | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee7b2fd5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
index ce968ce..fefd973 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
@@ -34,20 +34,20 @@ The following configuration properties are supported. See 
below for details.
 
 `resource-types.xml`
 
-| Configuration Property | Value | Description |
-|: |: |: |
+| Configuration Property | Description |
+|: |: |
 | `yarn.resource-types` | Comma-separated list of additional resources. May 
not include `memory`, `memory-mb`, or `vcores` |
 | `yarn.resource-types..units` | Default unit for the specified 
resource type |
 | `yarn.resource-types..minimum` | The minimum request for the 
specified resource type |
 | `yarn.resource-types..maximum` | The maximum request for the 
specified resource type |
 
-`node­-resources.xml`
+`node-resources.xml`
 
-| Configuration Property | Value | Description |
-|: |: |: |
+| Configuration Property | Description |
+|: |: |
 | `yarn.nodemanager.resource-type.` | The count of the specified 
resource available from the node manager |
 
-Please note that the `resource-types.xml` and `node­-resources.xml` files
+Please note that the `resource-types.xml` and `node-resources.xml` files
 also need to be placed in the same configuration directory as `yarn-site.xml` 
if
 they are used. Alternatively, the properties may be placed into the
 `yarn-site.xml` file instead.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-8288. Fix wrong number of table columns in Resource Model doc. Contributed by Weiwei Yang.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 e6c607cc2 -> 26e8fdffd


YARN-8288. Fix wrong number of table columns in Resource Model doc. Contributed 
by Weiwei Yang.

(cherry picked from commit 8a2b5914f3a68148f40f99105acf5dafcc326e89)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26e8fdff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26e8fdff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26e8fdff

Branch: refs/heads/branch-3.1
Commit: 26e8fdffd623b508b22ee21b8e8600e23ae4c7e7
Parents: e6c607c
Author: Naganarasimha 
Authored: Tue May 15 00:03:38 2018 +0800
Committer: Naganarasimha 
Committed: Tue May 15 00:10:58 2018 +0800

--
 .../hadoop-yarn-site/src/site/markdown/ResourceModel.md | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/26e8fdff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
index 75e5c92..f968b5f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
@@ -45,20 +45,20 @@ The following configuration properties are supported. See 
below for details.
 
 `resource-types.xml`
 
-| Configuration Property | Value | Description |
-|: |: |: |
+| Configuration Property | Description |
+|: |: |
 | `yarn.resource-types` | Comma-separated list of additional resources. May 
not include `memory`, `memory-mb`, or `vcores` |
 | `yarn.resource-types..units` | Default unit for the specified 
resource type |
 | `yarn.resource-types..minimum` | The minimum request for the 
specified resource type |
 | `yarn.resource-types..maximum` | The maximum request for the 
specified resource type |
 
-`node­-resources.xml`
+`node-resources.xml`
 
-| Configuration Property | Value | Description |
-|: |: |: |
+| Configuration Property | Description |
+|: |: |
 | `yarn.nodemanager.resource-type.` | The count of the specified 
resource available from the node manager |
 
-Please note that the `resource-types.xml` and `node­-resources.xml` files
+Please note that the `resource-types.xml` and `node-resources.xml` files
 also need to be placed in the same configuration directory as `yarn-site.xml` 
if
 they are used. Alternatively, the properties may be placed into the
 `yarn-site.xml` file instead.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

hadoop git commit: YARN-8288. Fix wrong number of table columns in Resource Model doc. Contributed by Weiwei Yang.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 89d0b87ad -> 8a2b5914f


YARN-8288. Fix wrong number of table columns in Resource Model doc. Contributed 
by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a2b5914
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a2b5914
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a2b5914

Branch: refs/heads/trunk
Commit: 8a2b5914f3a68148f40f99105acf5dafcc326e89
Parents: 89d0b87
Author: Naganarasimha 
Authored: Tue May 15 00:03:38 2018 +0800
Committer: Naganarasimha 
Committed: Tue May 15 00:05:23 2018 +0800

--
 .../hadoop-yarn-site/src/site/markdown/ResourceModel.md | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a2b5914/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
index 75e5c92..f968b5f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceModel.md
@@ -45,20 +45,20 @@ The following configuration properties are supported. See 
below for details.
 
 `resource-types.xml`
 
-| Configuration Property | Value | Description |
-|: |: |: |
+| Configuration Property | Description |
+|: |: |
 | `yarn.resource-types` | Comma-separated list of additional resources. May 
not include `memory`, `memory-mb`, or `vcores` |
 | `yarn.resource-types..units` | Default unit for the specified 
resource type |
 | `yarn.resource-types..minimum` | The minimum request for the 
specified resource type |
 | `yarn.resource-types..maximum` | The maximum request for the 
specified resource type |
 
-`node­-resources.xml`
+`node-resources.xml`
 
-| Configuration Property | Value | Description |
-|: |: |: |
+| Configuration Property | Description |
+|: |: |
 | `yarn.nodemanager.resource-type.` | The count of the specified 
resource available from the node manager |
 
-Please note that the `resource-types.xml` and `node­-resources.xml` files
+Please note that the `resource-types.xml` and `node-resources.xml` files
 also need to be placed in the same configuration directory as `yarn-site.xml` 
if
 they are used. Alternatively, the properties may be placed into the
 `yarn-site.xml` file instead.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[48/50] [abbrv] hadoop git commit: YARN-7856. Validate Node Attributes from NM. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7856. Validate Node Attributes from NM. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebb4b6e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebb4b6e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebb4b6e9

Branch: refs/heads/YARN-3409
Commit: ebb4b6e90cc443b7be82fe05a481157e8ecdd27e
Parents: 9a1f1bf
Author: Sunil G 
Authored: Tue Feb 27 08:15:42 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  |  2 ++
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   | 31 
 .../ScriptBasedNodeAttributesProvider.java  | 25 ++--
 .../TestScriptBasedNodeAttributesProvider.java  | 27 +
 4 files changed, 83 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb4b6e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 01c70b2..4f6846b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.yarn.util.Records;
 public abstract class NodeAttribute {
 
   public static final String DEFAULT_PREFIX = "";
+  public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
+  public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
 
   public static NodeAttribute newInstance(String attributeName,
   NodeAttributeType attributeType, String attributeValue) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb4b6e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
index d918712..fdfd0ce 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -17,7 +17,11 @@
  */
 package org.apache.hadoop.yarn.nodelabels;
 
+import com.google.common.base.Strings;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
 import java.io.IOException;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 /**
@@ -94,4 +98,31 @@ public final class NodeLabelUtil {
   + ", now it is= " + prefix);
 }
   }
+
+  /**
+   * Validate if a given set of attributes are valid. Attributes could be
+   * invalid if any of following conditions is met:
+   *
+   * 
+   *   Missing prefix: the attribute doesn't have prefix defined
+   *   Malformed attribute prefix: the prefix is not in valid format
+   * 
+   * @param attributeSet
+   * @throws IOException
+   */
+  public static void validateNodeAttributes(Set attributeSet)
+  throws IOException {
+if (attributeSet != null && !attributeSet.isEmpty()) {
+  for (NodeAttribute nodeAttribute : attributeSet) {
+String prefix = nodeAttribute.getAttributePrefix();
+if (Strings.isNullOrEmpty(prefix)) {
+  throw new IOException("Attribute prefix must be set");
+}
+// Verify attribute prefix format.
+checkAndThrowAttributePrefix(prefix);
+// Verify attribute name format.
+checkAndThrowLabelName(nodeAttribute.getAttributeName());
+  }
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb4b6e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
 

[17/50] [abbrv] hadoop git commit: HADOOP-13756 LocalMetadataStore#put(DirListingMetadata) should also put file metadata into fileHash. (Gabor Bota)

[naganarasimha_gr]

HADOOP-13756 LocalMetadataStore#put(DirListingMetadata) should also put file 
metadata into fileHash. (Gabor Bota)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d8bcf53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d8bcf53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d8bcf53

Branch: refs/heads/YARN-3409
Commit: 7d8bcf534acce52fb8cfb745a8671f9350d3b5be
Parents: 0a293de
Author: Aaron Fabbri 
Authored: Thu Apr 26 20:40:30 2018 -0700
Committer: Aaron Fabbri 
Committed: Thu Apr 26 20:41:29 2018 -0700

--
 .../fs/s3a/s3guard/LocalMetadataStore.java  |  1 +
 .../fs/s3a/s3guard/MetadataStoreTestBase.java   | 27 
 2 files changed, 28 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d8bcf53/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index 306428f..0061b56 100644
--- 
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ 
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -281,6 +281,7 @@ public class LocalMetadataStore implements MetadataStore {
   LOG.debug("put dirMeta {}", meta.prettyPrint());
 }
 dirHash.put(standardize(meta.getPath()), meta);
+put(meta.getListing());
   }
 
   public synchronized void put(Collection metas) throws

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d8bcf53/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
--
diff --git 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 478c2e8..806940b 100644
--- 
a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ 
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -738,6 +738,33 @@ public abstract class MetadataStoreTestBase extends Assert 
{
 }
   }
 
+  @Test
+  public void testPutDirListingMetadataPutsFileMetadata()
+  throws IOException {
+boolean authoritative = true;
+String[] filenames = {"/dir1/file1", "/dir1/file2", "/dir1/file3"};
+String dirPath = "/dir1";
+
+ArrayList metas = new ArrayList<>(filenames.length);
+for (String filename : filenames) {
+  metas.add(new PathMetadata(makeFileStatus(filename, 100)));
+}
+DirListingMetadata dirMeta =
+new DirListingMetadata(strToPath(dirPath), metas, authoritative);
+ms.put(dirMeta);
+
+if (!allowMissing()) {
+  assertDirectorySize(dirPath, filenames.length);
+  PathMetadata metadata;
+  for(String fileName : filenames){
+metadata = ms.get(strToPath(fileName));
+assertNotNull(String.format(
+"PathMetadata for file %s should not be null.", fileName),
+metadata);
+  }
+}
+  }
+
   /*
* Helper functions.
*/


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[07/50] [abbrv] hadoop git commit: HADOOP-15404. Remove multibyte characters in DataNodeUsageReportUtil

[naganarasimha_gr]

HADOOP-15404. Remove multibyte characters in DataNodeUsageReportUtil

Signed-off-by: Akira Ajisaka 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f655aed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f655aed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f655aed

Branch: refs/heads/YARN-3409
Commit: 6f655aed3b83899163740c98bac8d4919d65cde3
Parents: 48269c3
Author: Takanobu Asanuma 
Authored: Wed Apr 25 17:01:14 2018 +0900
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../hadoop/hdfs/server/protocol/DataNodeUsageReportUtil.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f655aed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DataNodeUsageReportUtil.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DataNodeUsageReportUtil.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DataNodeUsageReportUtil.java
index 3e90752..112d772 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DataNodeUsageReportUtil.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/DataNodeUsageReportUtil.java
@@ -23,7 +23,7 @@ import org.apache.hadoop.util.Time;
 
 /**
  * This class is helper class to generate a live usage report by calculating
- * the delta between  current DataNode usage metrics and the usage metrics
+ * the delta between current DataNode usage metrics and the usage metrics
  * captured at the time of the last report.
  */
 @InterfaceAudience.Private


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[37/50] [abbrv] hadoop git commit: YARN-7757. Refactor NodeLabelsProvider to be more generic and reusable for node attributes providers. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7757. Refactor NodeLabelsProvider to be more generic and reusable for node 
attributes providers. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf0f27b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf0f27b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf0f27b5

Branch: refs/heads/YARN-3409
Commit: cf0f27b5cb5593c46f74d460bcb5646e5153fa7b
Parents: a9782f8
Author: Naganarasimha 
Authored: Mon Feb 5 05:47:02 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:00:50 2018 +0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  33 +++
 .../src/main/resources/yarn-default.xml |  38 
 .../nodemanager/NodeStatusUpdaterImpl.java  |   4 +-
 .../AbstractNodeDescriptorsProvider.java| 197 
 .../nodelabels/AbstractNodeLabelsProvider.java  | 149 -
 .../ConfigurationNodeLabelsProvider.java|  13 +-
 .../nodelabels/NodeAttributesProvider.java  |  34 +++
 .../nodelabels/NodeDescriptorsProvider.java |  45 
 .../nodelabels/NodeDescriptorsScriptRunner.java |  84 +++
 .../nodelabels/NodeLabelsProvider.java  |  22 +-
 .../ScriptBasedNodeAttributesProvider.java  | 129 +++
 .../ScriptBasedNodeLabelsProvider.java  | 126 +++
 .../nodemanager/nodelabels/package-info.java|  28 +++
 .../TestNodeStatusUpdaterForLabels.java |  39 ++--
 .../TestConfigurationNodeLabelsProvider.java|  26 ++-
 .../TestScriptBasedNodeAttributesProvider.java  | 223 +++
 .../TestScriptBasedNodeLabelsProvider.java  |  18 +-
 17 files changed, 910 insertions(+), 298 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf0f27b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8aa136d..b908faa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3463,6 +3463,9 @@ public class YarnConfiguration extends Configuration {
   private static final String NM_NODE_LABELS_PREFIX = NM_PREFIX
   + "node-labels.";
 
+  private static final String NM_NODE_ATTRIBUTES_PREFIX = NM_PREFIX
+  + "node-attributes.";
+
   public static final String NM_NODE_LABELS_PROVIDER_CONFIG =
   NM_NODE_LABELS_PREFIX + "provider";
 
@@ -3473,6 +3476,9 @@ public class YarnConfiguration extends Configuration {
   private static final String NM_NODE_LABELS_PROVIDER_PREFIX =
   NM_NODE_LABELS_PREFIX + "provider.";
 
+  private static final String NM_NODE_ATTRIBUTES_PROVIDER_PREFIX =
+  NM_NODE_ATTRIBUTES_PREFIX + "provider.";
+
   public static final String NM_NODE_LABELS_RESYNC_INTERVAL =
   NM_NODE_LABELS_PREFIX + "resync-interval-ms";
 
@@ -3543,6 +3549,33 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
   NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
 
+  /**
+   * Node attribute provider fetch attributes interval and timeout.
+   */
+  public static final String NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS =
+  NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "fetch-interval-ms";
+
+  public static final long
+  DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS = 10 * 60 * 1000;
+
+  public static final String NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS =
+  NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "fetch-timeout-ms";
+
+  public static final long DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS
+  = DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS * 2;
+
+  /**
+   * Script to collect node attributes.
+   */
+  private static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX =
+  NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "script.";
+
+  public static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH =
+  NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX + "path";
+
+  public static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS =
+  NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX + "opts";
+
   /*
* Support to view apps for given user in secure cluster.
*/

[20/50] [abbrv] hadoop git commit: YARN-8004. Add unit tests for inter queue preemption for dominant resource calculator. Contributed by Zian Chen.

[naganarasimha_gr]

YARN-8004. Add unit tests for inter queue preemption for dominant resource 
calculator. Contributed by Zian Chen.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71220d21
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71220d21
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71220d21

Branch: refs/heads/YARN-3409
Commit: 71220d218db59cab0b03bbba427e5e9ef5b3003c
Parents: 7fc09c4
Author: Sunil G 
Authored: Fri Apr 27 10:42:30 2018 +0530
Committer: Sunil G 
Committed: Fri Apr 27 10:42:30 2018 +0530

--
 ...pacityPreemptionPolicyInterQueueWithDRF.java | 68 
 1 file changed, 68 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/71220d21/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
new file mode 100644
index 000..0d6d350
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyInterQueueWithDRF.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+
+import org.junit.Test;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class TestProportionalCapacityPreemptionPolicyInterQueueWithDRF
+extends ProportionalCapacityPreemptionPolicyMockFramework {
+  @Test
+  public void testInterQueuePreemptionWithMultipleResource()
+  throws Exception {
+/**
+ * Queue structure is:
+ *
+ * 
+ *   root
+ *   /  \
+ *  ab
+ * 
+ *
+ */
+
+String labelsConfig = "=100:200,true"; // default partition
+String nodesConfig = "n1="; // only one node
+String queuesConfig =
+// guaranteed,max,used,pending
+"root(=[100:200 100:200 100:200 100:200]);" + //root
+"-a(=[50:100 100:200 40:80 30:70]);" + // a
+"-b(=[50:100 100:200 60:120 40:50])";   // b
+
+String appsConfig =
+//queueName\t(priority,resource,host,expression,#repeat,reserved)
+"a\t(1,2:4,n1,,20,false);" + // app1 in a
+"b\t(1,2:4,n1,,30,false)"; // app2 in b
+
+buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig, true);
+policy.editSchedule();
+
+// Preemption should happen in Queue b, preempt <10,20> to Queue a
+verify(mDisp, never()).handle(argThat(
+new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+getAppAttemptId(1;
+verify(mDisp, times(5)).handle(argThat(
+new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+getAppAttemptId(2;
+  }
+}


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[47/50] [abbrv] hadoop git commit: YARN-8033. CLI Integration with NodeAttributesManagerImpl. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-8033. CLI Integration with NodeAttributesManagerImpl. Contributed by 
Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0346915d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0346915d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0346915d

Branch: refs/heads/YARN-3409
Commit: 0346915d15e98c67a775521de5ab3671fe97c8fe
Parents: 349e632
Author: bibinchundatt 
Authored: Sun Apr 1 19:24:00 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  |   3 +-
 .../yarn/client/cli/NodeAttributesCLI.java  |  18 +-
 .../yarn/client/cli/TestNodeAttributesCLI.java  |  18 ++
 .../server/resourcemanager/AdminService.java|  97 +++
 .../resourcemanager/TestRMAdminService.java | 170 +--
 5 files changed, 282 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0346915d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 4f6846b..25ac9ab 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -45,13 +45,12 @@ import org.apache.hadoop.yarn.util.Records;
 @Unstable
 public abstract class NodeAttribute {
 
-  public static final String DEFAULT_PREFIX = "";
   public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
   public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
 
   public static NodeAttribute newInstance(String attributeName,
   NodeAttributeType attributeType, String attributeValue) {
-return newInstance(DEFAULT_PREFIX, attributeName, attributeType,
+return newInstance(PREFIX_CENTRALIZED, attributeName, attributeType,
 attributeValue);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0346915d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
index 2eff155..df5a57d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.client.cli;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -50,6 +52,7 @@ import 
org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappin
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 
 /**
  * CLI to map attributes to Nodes.
@@ -311,7 +314,7 @@ public class NodeAttributesCLI extends Configured 
implements Tool {
*/
   private List buildNodeLabelsMapFromStr(String args,
   boolean validateForAttributes, AttributeMappingOperationType operation) {
-List nodeToAttributesList = new ArrayList<>();
+Map nodeToAttributesMap = new HashMap<>();
 for (String nodeToAttributesStr : args.split("[ \n]")) {
   // for each node to attribute mapping
   nodeToAttributesStr = nodeToAttributesStr.trim();
@@ -384,8 +387,9 @@ public class NodeAttributesCLI extends Configured 
implements Tool {
   // TODO when we support different type of attribute type we need to
   // cross verify whether input attributes itself is not violating
   // attribute Name to Type mapping.
-  attributesList.add(NodeAttribute.newInstance(attributeName.trim(),
-  attributeType, attributeValue.trim()));
+  attributesList
+  .add(NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED,
+  attributeName.trim(), 

[42/50] [abbrv] hadoop git commit: YARN-8094. Support configuration based Node Attribute provider. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-8094. Support configuration based Node Attribute provider. Contributed by 
Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/290c087a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/290c087a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/290c087a

Branch: refs/heads/YARN-3409
Commit: 290c087a6fe8b4224a14d306bc58108aea3e74e0
Parents: 4ef64ff
Author: Sunil G 
Authored: Sat Mar 31 19:53:06 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../ConfigurationNodeAttributesProvider.java| 70 +-
 .../ScriptBasedNodeAttributesProvider.java  |  8 +++
 ...TestConfigurationNodeAttributesProvider.java | 74 
 3 files changed, 150 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/290c087a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
index 74341eb..ab8a8b1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
@@ -18,13 +18,19 @@
 package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.EnumUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.TimerTask;
 import java.util.Set;
@@ -38,6 +44,9 @@ public class ConfigurationNodeAttributesProvider
   private static final Logger LOG =
   LoggerFactory.getLogger(ConfigurationNodeAttributesProvider.class);
 
+  private static final String NODE_ATTRIBUTES_DELIMITER = ":";
+  private static final String NODE_ATTRIBUTE_DELIMITER = ",";
+
   public ConfigurationNodeAttributesProvider() {
 super("Configuration Based Node Attributes Provider");
   }
@@ -59,11 +68,68 @@ public class ConfigurationNodeAttributesProvider
 setDescriptors(parseAttributes(configuredNodeAttributes));
   }
 
-  // TODO parse attributes from configuration
   @VisibleForTesting
   public Set parseAttributes(String config)
   throws IOException {
-return new HashSet<>();
+if (Strings.isNullOrEmpty(config)) {
+  return ImmutableSet.of();
+}
+Set attributeSet = new HashSet<>();
+// Configuration value should be in one line, format:
+// "ATTRIBUTE_NAME,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE",
+// multiple node-attributes are delimited by ":".
+// Each attribute str should not container any space.
+String[] attributeStrs = config.split(NODE_ATTRIBUTES_DELIMITER);
+for (String attributeStr : attributeStrs) {
+  String[] fields = attributeStr.split(NODE_ATTRIBUTE_DELIMITER);
+  if (fields.length != 3) {
+throw new IOException("Invalid value for "
++ YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
++ "=" + config);
+  }
+
+  // We don't allow user config to overwrite our dist prefix,
+  // so disallow any prefix set in the configuration.
+  if (fields[0].contains("/")) {
+throw new IOException("Node attribute set in "
++ YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
++ " should not contain any prefix.");
+  }
+
+  // Make sure attribute type is valid.
+  if (!EnumUtils.isValidEnum(NodeAttributeType.class, fields[1])) {
+throw new IOException("Invalid node attribute type: "
++ fields[1] + ", valid values are "
++ 

[35/50] [abbrv] hadoop git commit: YARN-6855. [YARN-3409] CLI Proto Modifications to support Node Attributes. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-6855. [YARN-3409] CLI Proto Modifications to support Node Attributes. 
Contributed by Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/366b863d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/366b863d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/366b863d

Branch: refs/heads/YARN-3409
Commit: 366b863d6b8170241239a0c26120998d12d8439d
Parents: 4844406
Author: Naganarasimha 
Authored: Sun Jan 21 00:53:02 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:00:50 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  |  78 
 .../yarn/api/records/NodeAttributeType.java |  35 
 .../ResourceManagerAdministrationProtocol.java  |  13 +-
 .../AttributeMappingOperationType.java  |  42 
 .../api/protocolrecords/NodeToAttributes.java   |  59 ++
 .../NodesToAttributesMappingRequest.java|  69 +++
 .../NodesToAttributesMappingResponse.java   |  27 +++
 ...esourcemanager_administration_protocol.proto |   1 +
 ..._server_resourcemanager_service_protos.proto |  21 ++
 .../src/main/proto/yarn_protos.proto|  11 ++
 .../records/impl/pb/NodeAttributePBImpl.java| 155 +++
 ...nagerAdministrationProtocolPBClientImpl.java |  26 ++-
 ...agerAdministrationProtocolPBServiceImpl.java |  31 ++-
 .../impl/pb/NodeToAttributesPBImpl.java | 161 +++
 .../NodesToAttributesMappingRequestPBImpl.java  | 194 +++
 .../NodesToAttributesMappingResponsePBImpl.java |  47 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |  34 +++-
 .../yarn/server/MockResourceManagerFacade.java  |  17 +-
 .../server/resourcemanager/AdminService.java|  10 +
 .../DefaultRMAdminRequestInterceptor.java   |   9 +
 .../router/rmadmin/RouterRMAdminService.java|  10 +
 .../PassThroughRMAdminRequestInterceptor.java   |   9 +
 22 files changed, 1044 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/366b863d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
new file mode 100644
index 000..13081f3
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * 
+ * Node Attribute is a kind of a label which represents one of the
+ * attribute/feature of a Node. Its different from node partition label as
+ * resource guarantees across the queues will not be maintained for these type
+ * of labels.
+ * 
+ * 
+ * A given Node can be mapped with any kind of attribute, few examples are
+ * HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS.
+ * 
+ * 
+ * Its not compulsory for all the attributes to have value, empty string is the
+ * default value of the NodeAttributeType.STRING
+ * 
+ *
+ */
+@Public
+@Unstable
+public abstract class NodeAttribute {
+
+  public static NodeAttribute newInstance(String attributeName,
+  NodeAttributeType attributeType, String attributeValue) {
+NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class);
+nodeAttribute.setAttributeName(attributeName);
+nodeAttribute.setAttributeType(attributeType);
+nodeAttribute.setAttributeValue(attributeValue);
+return nodeAttribute;
+  }
+
+  @Public
+  @Unstable
+  public 

[29/50] [abbrv] hadoop git commit: YARN-8210. AMRMClient logging on every heartbeat to track updation of AM RM token causes too many log lines to be generated in AM logs. (Suma Shivaprasad via wangda)

[naganarasimha_gr]

YARN-8210. AMRMClient logging on every heartbeat to track updation of AM RM 
token causes too many log lines to be generated in AM logs. (Suma Shivaprasad 
via wangda)

Change-Id: I70edd6e301fd5e78d479e1882aedc9332a0827aa


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1833d9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1833d9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1833d9b

Branch: refs/heads/YARN-3409
Commit: b1833d9ba2c078582161da45ac392dd5c361dcdf
Parents: ef3ecc3
Author: Wangda Tan 
Authored: Fri Apr 27 13:07:38 2018 -0700
Committer: Wangda Tan 
Committed: Fri Apr 27 13:07:38 2018 -0700

--
 .../java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java | 1 -
 1 file changed, 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1833d9b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index a8e4dfc..ef849b2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -,7 +,6 @@ public class AMRMClientImpl 
extends AMRMClient {
 // to ensure we replace the previous token setup by the RM.
 // Afterwards we can update the service address for the RPC layer.
 UserGroupInformation currentUGI = UserGroupInformation.getCurrentUser();
-LOG.info("Updating with new AMRMToken");
 currentUGI.addToken(amrmToken);
 amrmToken.setService(ClientRMProxy.getAMRMTokenService(getConfig()));
   }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[36/50] [abbrv] hadoop git commit: YARN-6856. [YARN-3409] Support CLI for Node Attributes Mapping. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-6856. [YARN-3409] Support CLI for Node Attributes Mapping. Contributed by 
Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ddad402
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ddad402
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ddad402

Branch: refs/heads/YARN-3409
Commit: 8ddad40218f8e73d893f3ee889215e52ff04745e
Parents: 366b863
Author: Naganarasimha 
Authored: Tue Jan 23 07:18:20 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:00:50 2018 +0800

--
 .../main/java/org/apache/hadoop/ha/HAAdmin.java |   2 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   5 +
 .../yarn/client/cli/NodeAttributesCLI.java  | 410 +++
 .../yarn/client/cli/TestNodeAttributesCLI.java  | 328 +++
 4 files changed, 744 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ddad402/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index 9b7d7ba..8c92bd0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -575,7 +575,7 @@ public abstract class HAAdmin extends Configured implements 
Tool {
 return 0;
   }
   
-  protected static class UsageInfo {
+  public static class UsageInfo {
 public final String args;
 public final String help;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ddad402/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 69afe6f..7cd838f 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -55,6 +55,7 @@ function hadoop_usage
   hadoop_add_subcommand "timelinereader" client "run the timeline reader 
server"
   hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
   hadoop_add_subcommand "top" client "view cluster information"
+  hadoop_add_subcommand "node-attributes" "map node to attibutes"
   hadoop_add_subcommand "version" client "print the version"
   hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
 }
@@ -186,6 +187,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
   hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
 ;;
+   node-attributes)
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false"
+  HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.NodeAttributesCLI'
+   ;;
 timelineserver)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ddad402/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
new file mode 100644
index 000..2eff155
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -0,0 +1,410 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 

[30/50] [abbrv] hadoop git commit: YARN-8005. Add unit tests for queue priority with dominant resource calculator. (Zian Chen via wangda)

[naganarasimha_gr]

YARN-8005. Add unit tests for queue priority with dominant resource calculator. 
(Zian Chen via wangda)

Change-Id: I17a645f20869a1e5d86fa7a325c93fec908b91dc


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92c53314
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92c53314
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92c53314

Branch: refs/heads/YARN-3409
Commit: 92c5331423afee172e843198f4824469eb92e89b
Parents: b1833d9
Author: Wangda Tan 
Authored: Fri Apr 27 13:08:43 2018 -0700
Committer: Wangda Tan 
Committed: Fri Apr 27 13:08:43 2018 -0700

--
 .../TestPreemptionForQueueWithPriorities.java   | 147 +++
 1 file changed, 147 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/92c53314/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
index 2b54d77..e9a8116 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestPreemptionForQueueWithPriorities.java
@@ -18,10 +18,16 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
 
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.mockito.Matchers.argThat;
 import static org.mockito.Mockito.never;
@@ -30,8 +36,28 @@ import static org.mockito.Mockito.verify;
 
 public class TestPreemptionForQueueWithPriorities
 extends ProportionalCapacityPreemptionPolicyMockFramework {
+  // Initialize resource map
+  private Map riMap = new HashMap<>();
+
   @Before
   public void setup() {
+
+// Initialize mandatory resources
+ResourceInformation memory = ResourceInformation.newInstance(
+ResourceInformation.MEMORY_MB.getName(),
+ResourceInformation.MEMORY_MB.getUnits(),
+YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ResourceInformation vcores = ResourceInformation.newInstance(
+ResourceInformation.VCORES.getName(),
+ResourceInformation.VCORES.getUnits(),
+YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+riMap.put(ResourceInformation.MEMORY_URI, memory);
+riMap.put(ResourceInformation.VCORES_URI, vcores);
+
+ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
 super.setup();
 policy = new ProportionalCapacityPreemptionPolicy(rmContext, cs, mClock);
   }
@@ -358,4 +384,125 @@ public class TestPreemptionForQueueWithPriorities
 getAppAttemptId(4;
   }
 
+  @Test
+  public void 
testPriorityPreemptionWithMandatoryResourceForHierarchicalOfQueues()
+  throws Exception {
+/**
+ * Queue structure is:
+ *
+ * 
+ *   root
+ *   /  \
+ *  ab
+ */  \  /  \
+ *   a1  a2 b1  b2
+ * 
+ *
+ * a2 is underserved and need more resource. b2 will be preemptable.
+ */
+
+String labelsConfig = "=100:200,true"; // default partition
+String nodesConfig = "n1="; // only one node
+String queuesConfig =
+// guaranteed,max,used,pending
+"root(=[100:200 100:200 100:200 100:200]);" + //root
+"-a(=[50:100 100:200 20:40 60:100]){priority=1};" + // a
+"--a1(=[10:20 100:200 10:30 30:20]){priority=1};" + // a1
+"--a2(=[40:80 100:200 10:10 30:80]){priority=1};" + // a2
+"-b(=[50:100 100:200 

[34/50] [abbrv] hadoop git commit: YARN-7840. Update PB for prefix support of node attributes. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-7840. Update PB for prefix support of node attributes. Contributed by 
Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9782f8d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9782f8d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9782f8d

Branch: refs/heads/YARN-3409
Commit: a9782f8de94432e02b1b0581b3396e228f21e258
Parents: 8ef1089
Author: bibinchundatt 
Authored: Fri Feb 2 10:31:00 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:00:50 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  | 22 ++-
 .../src/main/proto/yarn_protos.proto|  7 ++--
 .../records/impl/pb/NodeAttributePBImpl.java| 39 +---
 .../hadoop/yarn/api/TestPBImplRecords.java  |  7 ++--
 4 files changed, 61 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9782f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 13081f3..01c70b2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -37,15 +37,27 @@ import org.apache.hadoop.yarn.util.Records;
  * Its not compulsory for all the attributes to have value, empty string is the
  * default value of the NodeAttributeType.STRING
  * 
- *
+ * 
+ * Node Attribute Prefix is used as namespace to segregate the attributes.
+ * 
  */
 @Public
 @Unstable
 public abstract class NodeAttribute {
 
+  public static final String DEFAULT_PREFIX = "";
+
   public static NodeAttribute newInstance(String attributeName,
   NodeAttributeType attributeType, String attributeValue) {
+return newInstance(DEFAULT_PREFIX, attributeName, attributeType,
+attributeValue);
+  }
+
+  public static NodeAttribute newInstance(String attributePrefix,
+  String attributeName, NodeAttributeType attributeType,
+  String attributeValue) {
 NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class);
+nodeAttribute.setAttributePrefix(attributePrefix);
 nodeAttribute.setAttributeName(attributeName);
 nodeAttribute.setAttributeType(attributeType);
 nodeAttribute.setAttributeValue(attributeValue);
@@ -54,6 +66,14 @@ public abstract class NodeAttribute {
 
   @Public
   @Unstable
+  public abstract String getAttributePrefix();
+
+  @Public
+  @Unstable
+  public abstract void setAttributePrefix(String attributePrefix);
+
+  @Public
+  @Unstable
   public abstract String getAttributeName();
 
   @Public

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9782f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 41f5ccb..815e989 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -377,9 +377,10 @@ enum NodeAttributeTypeProto {
 }
 
 message NodeAttributeProto {
-  optional string attributeName = 1;
-  optional NodeAttributeTypeProto attributeType = 2;
-  optional string attributeValue = 3;
+  optional string attributePrefix = 1;
+  required string attributeName = 2;
+  optional NodeAttributeTypeProto attributeType = 3 [default = STRING];
+  optional string attributeValue = 4 [default=""];
 }
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9782f8d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 11c9c48..7810939 100644
--- 

[28/50] [abbrv] hadoop git commit: YARN-8221. RMWebServices also need to honor yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.

[naganarasimha_gr]

YARN-8221. RMWebServices also need to honor 
yarn.resourcemanager.display.per-user-apps. Contributed by Sunil G.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef3ecc30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef3ecc30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef3ecc30

Branch: refs/heads/YARN-3409
Commit: ef3ecc308dbea41c6a88bd4d16739c7bbc10cdda
Parents: bff3d7b
Author: Rohith Sharma K S 
Authored: Fri Apr 27 22:58:10 2018 +0530
Committer: Rohith Sharma K S 
Committed: Fri Apr 27 22:58:10 2018 +0530

--
 .../server/resourcemanager/webapp/RMWebServices.java   | 13 -
 1 file changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3ecc30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index d30764d..0564b67 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -228,6 +228,7 @@ public class RMWebServices extends WebServices implements 
RMWebServiceProtocol {
 
   @VisibleForTesting
   boolean isCentralizedNodeLabelConfiguration = true;
+  private boolean displayPerUserApps = false;
 
   public final static String DELEGATION_TOKEN_HEADER =
   "Hadoop-YARN-RM-Delegation-Token";
@@ -240,6 +241,9 @@ public class RMWebServices extends WebServices implements 
RMWebServiceProtocol {
 this.conf = conf;
 isCentralizedNodeLabelConfiguration =
 YarnConfiguration.isCentralizedNodeLabelConfiguration(conf);
+this.displayPerUserApps  = conf.getBoolean(
+YarnConfiguration.DISPLAY_APPS_FOR_LOGGED_IN_USER,
+YarnConfiguration.DEFAULT_DISPLAY_APPS_FOR_LOGGED_IN_USER);
   }
 
   RMWebServices(ResourceManager rm, Configuration conf,
@@ -608,7 +612,14 @@ public class RMWebServices extends WebServices implements 
RMWebServiceProtocol {
   DeSelectFields deSelectFields = new DeSelectFields();
   deSelectFields.initFields(unselectedFields);
 
-  AppInfo app = new AppInfo(rm, rmapp, hasAccess(rmapp, hsr),
+  boolean allowAccess = hasAccess(rmapp, hsr);
+  // Given RM is configured to display apps per user, skip apps to which
+  // this caller doesn't have access to view.
+  if (displayPerUserApps && !allowAccess) {
+continue;
+  }
+
+  AppInfo app = new AppInfo(rm, rmapp, allowAccess,
   WebAppUtils.getHttpSchemePrefix(conf), deSelectFields);
   allApps.add(app);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[31/50] [abbrv] hadoop git commit: HDFS-13434. RBF: Fix dead links in RBF document. Contributed by Chetna Chaudhari.

[naganarasimha_gr]

HDFS-13434. RBF: Fix dead links in RBF document. Contributed by Chetna 
Chaudhari.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f469628b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f469628b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f469628b

Branch: refs/heads/YARN-3409
Commit: f469628bba350ba79bc6a0d38f9dc1cb5eb65c77
Parents: 92c5331
Author: Inigo Goiri 
Authored: Fri Apr 27 15:13:47 2018 -0700
Committer: Inigo Goiri 
Committed: Fri Apr 27 15:13:47 2018 -0700

--
 .../src/site/markdown/HDFSRouterFederation.md   | 16 
 1 file changed, 8 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f469628b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
index 43e89ed..70c6226 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/site/markdown/HDFSRouterFederation.md
@@ -21,7 +21,7 @@ Introduction
 
 
 NameNodes have scalability limits because of the metadata overhead comprised 
of inodes (files and directories) and file blocks, the number of Datanode 
heartbeats, and the number of HDFS RPC client requests.
-The common solution is to split the filesystem into smaller subclusters [HDFS 
Federation](./Federation.html) and provide a federated view 
[ViewFs](./ViewFs.html).
+The common solution is to split the filesystem into smaller subclusters [HDFS 
Federation](../hadoop-hdfs/Federation.html) and provide a federated view 
[ViewFs](../hadoop-hdfs/ViewFs.html).
 The problem is how to maintain the split of the subclusters (e.g., namespace 
partition), which forces users to connect to multiple subclusters and manage 
the allocation of folders/files to them.
 
 
@@ -37,8 +37,8 @@ This layer must be scalable, highly available, and fault 
tolerant.
 
 This federation layer comprises multiple components.
 The _Router_ component that has the same interface as a NameNode, and forwards 
the client requests to the correct subcluster, based on ground-truth 
information from a State Store.
-The _State Store_ combines a remote _Mount Table_ (in the flavor of 
[ViewFs](./ViewFs.html), but shared between clients) and utilization 
(load/capacity) information about the subclusters.
-This approach has the same architecture as [YARN 
federation](../hadoop-yarn/Federation.html).
+The _State Store_ combines a remote _Mount Table_ (in the flavor of 
[ViewFs](../hadoop-hdfs/ViewFs.html), but shared between clients) and 
utilization (load/capacity) information about the subclusters.
+This approach has the same architecture as [YARN 
federation](../../hadoop-yarn/hadoop-yarn-site/Federation.html).
 
 ![Router-based Federation Sequence Diagram | 
width=800](./images/routerfederation.png)
 
@@ -140,7 +140,7 @@ Examples users may encounter include the following.
 ### Quota management
 Federation supports and controls global quota at mount table level.
 For performance reasons, the Router caches the quota usage and updates it 
periodically. These quota usage values
-will be used for quota-verification during each WRITE RPC call invoked in 
RouterRPCSever. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html)
+will be used for quota-verification during each WRITE RPC call invoked in 
RouterRPCSever. See [HDFS Quotas Guide](../hadoop-hdfs/HdfsQuotaAdminGuide.html)
 for the quota detail.
 
 ### State Store
@@ -163,7 +163,7 @@ The Routers discard the entries older than a certain 
threshold (e.g., ten Router
 
 * **Mount Table**:
 This table hosts the mapping between folders and subclusters.
-It is similar to the mount table in [ViewFs](.ViewFs.html) where it specifies 
the federated folder, the destination subcluster and the path in that folder.
+It is similar to the mount table in [ViewFs](../hadoop-hdfs/ViewFs.html) where 
it specifies the federated folder, the destination subcluster and the path in 
that folder.
 
 
 ### Security
@@ -175,7 +175,7 @@ Deployment
 
 By default, the Router is ready to take requests and monitor the NameNode in 
the local machine.
 It needs to know the State Store endpoint by setting 
`dfs.federation.router.store.driver.class`.
-The rest of the options are documented in 
[hdfs-default.xml](./hdfs-default.xml).
+The rest of the options are documented in 
[hdfs-default.xml](../hadoop-hdfs/hdfs-default.xml).
 
 Once the Router is configured, it can be started:
 
@@ -187,7 +187,7 @@ And to stop it:
 
 ### Mount table management
 

[19/50] [abbrv] hadoop git commit: YARN-8215. ATS v2 returns invalid YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS from NM. Contributed by Rohith Sharma K S.

[naganarasimha_gr]

YARN-8215. ATS v2 returns invalid YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS 
from NM. Contributed by Rohith Sharma K S.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fc09c4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fc09c4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fc09c4f

Branch: refs/heads/YARN-3409
Commit: 7fc09c4f7255645025ddf902dff70a17092ea039
Parents: 1634de0
Author: Sunil G 
Authored: Fri Apr 27 10:33:42 2018 +0530
Committer: Sunil G 
Committed: Fri Apr 27 10:33:42 2018 +0530

--
 .../timelineservice/NMTimelinePublisher.java | 15 ++-
 .../timelineservice/TestNMTimelinePublisher.java |  1 -
 2 files changed, 10 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fc09c4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 2ce3c5e..13d5c67 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -23,6 +23,8 @@ import java.security.PrivilegedExceptionAction;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -80,6 +82,7 @@ public class NMTimelinePublisher extends CompositeService {
   private NodeId nodeId;
 
   private String httpAddress;
+  private String httpPort;
 
   private UserGroupInformation nmLoginUGI;
 
@@ -101,6 +104,12 @@ public class NMTimelinePublisher extends CompositeService {
 UserGroupInformation.getLoginUser() :
 UserGroupInformation.getCurrentUser();
 LOG.info("Initialized NMTimelinePublisher UGI to " + nmLoginUGI);
+
+String webAppURLWithoutScheme =
+WebAppUtils.getNMWebAppURLWithoutScheme(conf);
+if (webAppURLWithoutScheme.contains(":")) {
+  httpPort = webAppURLWithoutScheme.split(":")[1];
+}
 super.serviceInit(conf);
   }
 
@@ -110,6 +119,7 @@ public class NMTimelinePublisher extends CompositeService {
 // context will be updated after containerManagerImpl is started
 // hence NMMetricsPublisher is added subservice of containerManagerImpl
 this.nodeId = context.getNodeId();
+this.httpAddress = nodeId.getHost() + ":" + httpPort;
   }
 
   @Override
@@ -331,11 +341,6 @@ public class NMTimelinePublisher extends CompositeService {
 
   public void publishContainerEvent(ContainerEvent event) {
 // publish only when the desired event is received
-if (this.httpAddress == null) {
-  // update httpAddress for first time. When this service started,
-  // web server will not be started.
-  this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort();
-}
 switch (event.getType()) {
 case INIT_CONTAINER:
   publishContainerCreatedEvent(event);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fc09c4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
index 0b8eaa9..43196c7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
+++ 

[50/50] [abbrv] hadoop git commit: YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-8100. Support API interface to query cluster attributes and attribute to 
nodes. Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13b0c115
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13b0c115
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13b0c115

Branch: refs/heads/YARN-3409
Commit: 13b0c1155418ee94bf250b57ea3ad2f989dd798f
Parents: f5ff4fa
Author: Naganarasimha 
Authored: Tue Apr 10 07:28:53 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/mapred/ResourceMgrDelegate.java  |  13 ++
 .../hadoop/mapred/TestClientRedirect.java   |  17 ++
 .../yarn/api/ApplicationClientProtocol.java |  37 
 .../GetAttributesToNodesRequest.java|  70 +++
 .../GetAttributesToNodesResponse.java   |  62 +++
 .../GetClusterNodeAttributesRequest.java|  47 +
 .../GetClusterNodeAttributesResponse.java   |  72 
 .../hadoop/yarn/conf/YarnConfiguration.java |   2 +-
 .../main/proto/applicationclient_protocol.proto |   2 +
 .../src/main/proto/yarn_protos.proto|   4 +
 .../src/main/proto/yarn_service_protos.proto|  15 ++
 .../hadoop/yarn/client/api/YarnClient.java  |  36 +++-
 .../yarn/client/api/impl/YarnClientImpl.java|  21 ++-
 .../ApplicationClientProtocolPBClientImpl.java  |  37 
 .../ApplicationClientProtocolPBServiceImpl.java |  44 +
 .../pb/GetAttributesToNodesRequestPBImpl.java   | 175 ++
 .../pb/GetAttributesToNodesResponsePBImpl.java  | 184 +++
 .../GetClusterNodeAttributesRequestPBImpl.java  |  75 
 .../GetClusterNodeAttributesResponsePBImpl.java | 156 
 .../yarn/nodelabels/NodeAttributesManager.java  |   9 +-
 .../hadoop/yarn/api/TestPBImplRecords.java  |  28 +++
 .../yarn/server/MockResourceManagerFacade.java  |  17 ++
 .../server/resourcemanager/ClientRMService.java |  30 +++
 .../nodelabels/NodeAttributesManagerImpl.java   |  33 ++--
 .../resourcemanager/TestClientRMService.java| 126 +
 .../DefaultClientRequestInterceptor.java|  17 ++
 .../clientrm/FederationClientInterceptor.java   |  17 ++
 .../router/clientrm/RouterClientRMService.java  |  19 ++
 .../PassThroughClientRequestInterceptor.java|  17 ++
 29 files changed, 1361 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b0c115/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index ac4b73b..4f96a6b 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -538,4 +539,16 @@ public class ResourceMgrDelegate extends YarnClient {
   throws YarnException, IOException {
 return client.getResourceTypeInfo();
   }
+
+  @Override
+  public Set getClusterAttributes()
+  throws YarnException, IOException {
+return client.getClusterAttributes();
+  }
+
+  @Override
+  public Map getAttributesToNodes(
+  Set attributes) throws YarnException, IOException {
+return client.getAttributesToNodes(attributes);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b0c115/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
 

[25/50] [abbrv] hadoop git commit: YARN-8204. Added a flag to disable YARN service upgrade. Contributed by Chandni Singh

[naganarasimha_gr]

YARN-8204.  Added a flag to disable YARN service upgrade.
Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14b47990
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14b47990
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14b47990

Branch: refs/heads/YARN-3409
Commit: 14b47990af39de71b0a09d995208f45ea3b79c23
Parents: 9ab3f97
Author: Eric Yang 
Authored: Fri Apr 27 12:23:56 2018 -0400
Committer: Eric Yang 
Committed: Fri Apr 27 12:24:43 2018 -0400

--
 .../yarn/service/client/ServiceClient.java  |  7 +
 .../yarn/service/conf/YarnServiceConf.java  |  7 +
 .../yarn/service/exceptions/ErrorStrings.java   |  2 ++
 .../yarn/service/TestYarnNativeServices.java|  1 +
 .../yarn/service/client/TestServiceClient.java  | 28 +---
 5 files changed, 41 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b47990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 52cd369..8dd5342 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -74,6 +74,7 @@ import 
org.apache.hadoop.yarn.service.containerlaunch.ClasspathConstructor;
 import org.apache.hadoop.yarn.service.containerlaunch.JavaCommandLineBuilder;
 import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
 import org.apache.hadoop.yarn.service.exceptions.BadConfigException;
+import org.apache.hadoop.yarn.service.exceptions.ErrorStrings;
 import org.apache.hadoop.yarn.service.exceptions.SliderException;
 import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
 import org.apache.hadoop.yarn.service.provider.ProviderUtils;
@@ -224,6 +225,12 @@ public class ServiceClient extends AppAdminClient 
implements SliderExitCodes,
 
   public int initiateUpgrade(Service service) throws YarnException,
   IOException {
+boolean upgradeEnabled = getConfig().getBoolean(
+YARN_SERVICE_UPGRADE_ENABLED,
+YARN_SERVICE_UPGRADE_ENABLED_DEFAULT);
+if (!upgradeEnabled) {
+  throw new YarnException(ErrorStrings.SERVICE_UPGRADE_DISABLED);
+}
 Service persistedService =
 ServiceApiUtil.loadService(fs, service.getName());
 if (!StringUtils.isEmpty(persistedService.getId())) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14b47990/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 55a3d70..13ed1aa 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -128,6 +128,13 @@ public class YarnServiceConf {
   YARN_SERVICE_PREFIX + "container-health-threshold.";
 
   /**
+   * Upgrade feature enabled for services.
+   */
+  public static final String YARN_SERVICE_UPGRADE_ENABLED =
+  "yarn.service.upgrade.enabled";
+  public static final boolean YARN_SERVICE_UPGRADE_ENABLED_DEFAULT = false;
+
+  /**
* The container health threshold percent when explicitly set for a specific
* component or globally for all components, will schedule a health check
* monitor to periodically check for the 

[49/50] [abbrv] hadoop git commit: YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b0c115/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
index 5ce4803..f6adb43 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
@@ -39,8 +39,12 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@@ -327,6 +331,19 @@ public class DefaultClientRequestInterceptor
 return clientRMProxy.getResourceTypeInfo(request);
   }
 
+  @Override
+  public GetAttributesToNodesResponse getAttributesToNodes(
+  GetAttributesToNodesRequest request) throws YarnException, IOException {
+return clientRMProxy.getAttributesToNodes(request);
+  }
+
+  @Override
+  public GetClusterNodeAttributesResponse getClusterNodeAttributes(
+  GetClusterNodeAttributesRequest request)
+  throws YarnException, IOException {
+return clientRMProxy.getClusterNodeAttributes(request);
+  }
+
   @VisibleForTesting
   public void setRMClient(ApplicationClientProtocol clientRM) {
 this.clientRMProxy = clientRM;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/13b0c115/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index 07eaf97..781ccfc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -44,8 +44,12 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@@ -732,4 +736,17 @@ public class FederationClientInterceptor
   

[46/50] [abbrv] hadoop git commit: YARN-7988. Refactor FSNodeLabelStore code for Node Attributes store support. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-7988. Refactor FSNodeLabelStore code for Node Attributes store support. 
Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ef64ffa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ef64ffa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ef64ffa

Branch: refs/heads/YARN-3409
Commit: 4ef64ffa84735ab128085c52068f60191ea86b1c
Parents: d4112d7
Author: Sunil G 
Authored: Thu Mar 29 17:12:08 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../nodelabels/CommonNodeLabelsManager.java |   8 +-
 .../nodelabels/FileSystemNodeLabelsStore.java   | 268 +++
 .../hadoop/yarn/nodelabels/NodeLabelsStore.java |  32 +--
 .../NonAppendableFSNodeLabelStore.java  |  46 ++--
 .../nodelabels/store/AbstractFSNodeStore.java   | 213 +++
 .../yarn/nodelabels/store/FSStoreOpHandler.java | 108 
 .../hadoop/yarn/nodelabels/store/StoreOp.java   |  49 
 .../nodelabels/store/op/AddClusterLabelOp.java  |  73 +
 .../nodelabels/store/op/FSNodeStoreLogOp.java   |  35 +++
 .../nodelabels/store/op/NodeLabelMirrorOp.java  |  82 ++
 .../yarn/nodelabels/store/op/NodeToLabelOp.java |  75 ++
 .../store/op/RemoveClusterLabelOp.java  |  75 ++
 .../yarn/nodelabels/store/package-info.java |  21 ++
 ...emoveFromClusterNodeLabelsRequestPBImpl.java |  21 +-
 .../DummyCommonNodeLabelsManager.java   |   8 +-
 .../TestFileSystemNodeLabelsStore.java  |  16 +-
 .../nodelabels/NullRMNodeLabelsManager.java |   7 +
 .../webapp/TestRMWebServicesNodeLabels.java |   2 +-
 18 files changed, 845 insertions(+), 294 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ef64ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 24ee028..4973611 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -234,7 +234,10 @@ public class CommonNodeLabelsManager extends 
AbstractService {
 return initNodeLabelStoreInProgress;
   }
 
-  boolean isCentralizedConfiguration() {
+  /**
+   * @return true if node label configuration type is not distributed.
+   */
+  public boolean isCentralizedConfiguration() {
 return isCentralizedNodeLabelConfiguration;
   }
 
@@ -245,8 +248,7 @@ public class CommonNodeLabelsManager extends 
AbstractService {
 
conf.getClass(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS,
 FileSystemNodeLabelsStore.class, NodeLabelsStore.class),
 conf);
-this.store.setNodeLabelsManager(this);
-this.store.init(conf);
+this.store.init(conf, this);
 this.store.recover();
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ef64ffa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
index 0ec4ea4..e11e6f8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
@@ -18,275 +18,89 @@
 
 package org.apache.hadoop.yarn.nodelabels;
 
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
 import 

[18/50] [abbrv] hadoop git commit: YARN-8205. Application State is not updated to ATS if AM launching is delayed. Contributed by Rohith Sharma K S.

[naganarasimha_gr]

YARN-8205. Application State is not updated to ATS if AM launching is delayed. 
Contributed by Rohith Sharma K S.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1634de0f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1634de0f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1634de0f

Branch: refs/heads/YARN-3409
Commit: 1634de0fc1430d86b7688d16259a81462fba482f
Parents: 7d8bcf5
Author: Sunil G 
Authored: Fri Apr 27 10:26:57 2018 +0530
Committer: Sunil G 
Committed: Fri Apr 27 10:26:57 2018 +0530

--
 .../server/resourcemanager/metrics/TimelineServiceV1Publisher.java | 2 ++
 .../server/resourcemanager/metrics/TimelineServiceV2Publisher.java | 2 ++
 .../server/resourcemanager/metrics/TestSystemMetricsPublisher.java | 1 +
 .../resourcemanager/metrics/TestSystemMetricsPublisherForV2.java   | 1 +
 4 files changed, 6 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1634de0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index 73bb301..31b07f1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -108,6 +108,8 @@ public class TimelineServiceV1Publisher extends 
AbstractSystemMetricsPublisher {
 app.getApplicationSubmissionContext().getAMContainerSpec();
 entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND,
 amContainerSpec.getCommands());
+entityInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
+RMServerUtils.createApplicationState(app.getState()).toString());
 
 entity.setOtherInfo(entityInfo);
 TimelineEvent tEvent = new TimelineEvent();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1634de0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 7eaa6e7..90ecc75 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -139,6 +139,8 @@ public class TimelineServiceV2Publisher extends 
AbstractSystemMetricsPublisher {
 app.getApplicationSubmissionContext().getAMContainerSpec();
 entityInfo.put(ApplicationMetricsConstants.AM_CONTAINER_LAUNCH_COMMAND,
 amContainerSpec.getCommands());
+entityInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
+RMServerUtils.createApplicationState(app.getState()).toString());
 
 entity.setInfo(entityInfo);
 TimelineEvent tEvent = new TimelineEvent();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1634de0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
 

[41/50] [abbrv] hadoop git commit: YARN-6858. Attribute Manager to store and provide node attributes in RM. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-6858. Attribute Manager to store and provide node attributes in RM. 
Contributed by Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a1f1bf9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a1f1bf9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a1f1bf9

Branch: refs/heads/YARN-3409
Commit: 9a1f1bf90b6033a003a4ba0e5bcd94555ab759b2
Parents: cf0f27b
Author: Sunil G 
Authored: Fri Feb 23 08:01:58 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../records/impl/pb/NodeAttributePBImpl.java|  15 +-
 .../hadoop/yarn/nodelabels/AbstractLabel.java   |  71 +++
 .../AttributeExpressionOperation.java   |  26 +
 .../hadoop/yarn/nodelabels/AttributeValue.java  |  53 ++
 .../nodelabels/CommonNodeLabelsManager.java |  24 +-
 .../yarn/nodelabels/NodeAttributesManager.java  |  99 
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   |  97 
 .../hadoop/yarn/nodelabels/RMNodeAttribute.java | 104 
 .../hadoop/yarn/nodelabels/RMNodeLabel.java | 109 ++--
 .../yarn/nodelabels/StringAttributeValue.java   |  61 +++
 .../nodemanager/NodeStatusUpdaterImpl.java  |  16 +-
 .../resourcemanager/RMActiveServiceContext.java |  14 +
 .../yarn/server/resourcemanager/RMContext.java  |   5 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  13 +-
 .../nodelabels/NodeAttributesManagerImpl.java   | 527 +++
 .../nodelabels/NodeAttributesStoreEvent.java|  51 ++
 .../NodeAttributesStoreEventType.java   |  26 +
 18 files changed, 1220 insertions(+), 102 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a1f1bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 7810939..12a0ecc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
@@ -120,7 +120,13 @@ public class NodeAttributePBImpl extends NodeAttribute {
 
   @Override
   public int hashCode() {
-return getProto().hashCode();
+final int prime = 31;
+int result = 1;
+result = prime * result + ((getAttributePrefix() == null) ? 0
+: getAttributePrefix().hashCode());
+result = prime * result
++ ((getAttributeName() == null) ? 0 : getAttributeName().hashCode());
+return result;
   }
 
   @Override
@@ -133,13 +139,10 @@ public class NodeAttributePBImpl extends NodeAttribute {
 }
 if (obj instanceof NodeAttribute) {
   NodeAttribute other = (NodeAttribute) obj;
-  if (!compare(getAttributeName(), other.getAttributeName())) {
-return false;
-  }
-  if (!compare(getAttributeValue(), other.getAttributeValue())) {
+  if (!compare(getAttributePrefix(), other.getAttributePrefix())) {
 return false;
   }
-  if (!compare(getAttributeType(), other.getAttributeType())) {
+  if (!compare(getAttributeName(), other.getAttributeName())) {
 return false;
   }
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a1f1bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
new file mode 100644
index 000..6a44574
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of 

[27/50] [abbrv] hadoop git commit: HADOOP-15382. Log kinit output in credential renewal thread. Contributed by Gabor Bota.

[naganarasimha_gr]

HADOOP-15382. Log kinit output in credential renewal thread. Contributed by 
Gabor Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bff3d7b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bff3d7b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bff3d7b0

Branch: refs/heads/YARN-3409
Commit: bff3d7b0cf073ccc061db30af6d52fa4a9f21c05
Parents: 24a5ccb
Author: Wei-Chiu Chuang 
Authored: Fri Apr 27 10:05:55 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Fri Apr 27 10:05:55 2018 -0700

--
 .../java/org/apache/hadoop/security/UserGroupInformation.java| 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff3d7b0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index a9f6cb6..cb132b3 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -866,9 +866,9 @@ public class UserGroupInformation {
 if (now < nextRefresh) {
   Thread.sleep(nextRefresh - now);
 }
-Shell.execCommand(cmd, "-R");
+String output = Shell.execCommand(cmd, "-R");
 if (LOG.isDebugEnabled()) {
-  LOG.debug("renewed ticket");
+  LOG.debug("Renewed ticket. kinit output: {}", output);
 }
 reloginFromTicketCache();
 tgt = getTGT();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[33/50] [abbrv] hadoop git commit: YARN-7842. PB changes to carry node-attributes in NM heartbeat. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7842. PB changes to carry node-attributes in NM heartbeat. Contributed by 
Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ef10892
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ef10892
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ef10892

Branch: refs/heads/YARN-3409
Commit: 8ef1089258da005a292ef95b7d50cab6ac76db33
Parents: 8ddad40
Author: Weiwei Yang 
Authored: Wed Jan 31 20:28:41 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:00:50 2018 +0800

--
 .../protocolrecords/NodeHeartbeatRequest.java   | 17 +++
 .../impl/pb/NodeHeartbeatRequestPBImpl.java | 52 
 .../yarn_server_common_service_protos.proto |  5 ++
 .../protocolrecords/TestProtocolRecords.java| 12 +
 4 files changed, 86 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ef10892/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
index f238f79..4f99225 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
@@ -28,6 +28,7 @@ import 
org.apache.hadoop.yarn.server.api.records.AppCollectorData;
 import org.apache.hadoop.yarn.server.api.records.MasterKey;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 
 public abstract class NodeHeartbeatRequest {
   
@@ -61,6 +62,18 @@ public abstract class NodeHeartbeatRequest {
 return nodeHeartbeatRequest;
   }
 
+  public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
+  MasterKey lastKnownContainerTokenMasterKey,
+  MasterKey lastKnownNMTokenMasterKey, Set nodeLabels,
+  Set nodeAttributes,
+  Map registeringCollectors) {
+NodeHeartbeatRequest request = NodeHeartbeatRequest
+.newInstance(nodeStatus, lastKnownContainerTokenMasterKey,
+lastKnownNMTokenMasterKey, nodeLabels, registeringCollectors);
+request.setNodeAttributes(nodeAttributes);
+return request;
+  }
+
   public abstract NodeStatus getNodeStatus();
   public abstract void setNodeStatus(NodeStatus status);
 
@@ -85,4 +98,8 @@ public abstract class NodeHeartbeatRequest {
 
   public abstract void setRegisteringCollectors(Map appCollectorsMap);
+
+  public abstract Set getNodeAttributes();
+
+  public abstract void setNodeAttributes(Set nodeAttributes);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ef10892/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index 1ffd223..c59127a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -27,6 +27,9 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos;
 import 

[06/50] [abbrv] hadoop git commit: YARN-8183. Fix ConcurrentModificationException inside RMAppAttemptMetrics#convertAtomicLongMaptoLongMap. (Suma Shivaprasad via wangda)

[naganarasimha_gr]

YARN-8183. Fix ConcurrentModificationException inside 
RMAppAttemptMetrics#convertAtomicLongMaptoLongMap. (Suma Shivaprasad via wangda)

Change-Id: I347871d672001653a3afe2e99adefd74e0d798cd


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebedd69e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebedd69e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebedd69e

Branch: refs/heads/YARN-3409
Commit: ebedd69e254e5af8544364088cab4f477fa1fab5
Parents: 3b1dad9
Author: Wangda Tan 
Authored: Tue Apr 24 17:42:17 2018 -0700
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java  | 9 +
 1 file changed, 5 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebedd69e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 015cff7..4e75505 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -20,6 +20,7 @@ package 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -53,8 +54,8 @@ public class RMAppAttemptMetrics {
   
   private ReadLock readLock;
   private WriteLock writeLock;
-  private Map resourceUsageMap = new HashMap<>();
-  private Map preemptedResourceMap = new HashMap<>();
+  private Map resourceUsageMap = new ConcurrentHashMap<>();
+  private Map preemptedResourceMap = new 
ConcurrentHashMap<>();
   private RMContext rmContext;
 
   private int[][] localityStatistics =
@@ -97,7 +98,7 @@ public class RMAppAttemptMetrics {
   public Resource getResourcePreempted() {
 try {
   readLock.lock();
-  return resourcePreempted;
+  return Resource.newInstance(resourcePreempted);
 } finally {
   readLock.unlock();
 }
@@ -230,7 +231,7 @@ public class RMAppAttemptMetrics {
   }
 
   public Resource getApplicationAttemptHeadroom() {
-return applicationHeadroom;
+return Resource.newInstance(applicationHeadroom);
   }
 
   public void setApplicationAttemptHeadRoom(Resource headRoom) {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[03/50] [abbrv] hadoop git commit: YARN-8193. YARN RM hangs abruptly (stops allocating resources) when running successive applications. (Zian Chen via wangda)

[naganarasimha_gr]

YARN-8193. YARN RM hangs abruptly (stops allocating resources) when running 
successive applications. (Zian Chen via wangda)

Change-Id: Ia83dd2499ee9000b9e09ae5a932f21a13c0ddee6


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a0fa50f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a0fa50f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a0fa50f

Branch: refs/heads/YARN-3409
Commit: 2a0fa50f9d718d51036ccdf30c7d998362fe423b
Parents: 13d389b
Author: Wangda Tan 
Authored: Wed Apr 25 22:10:18 2018 -0700
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../allocator/RegularContainerAllocator.java| 42 +++-
 1 file changed, 31 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a0fa50f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index afa468b..99deb1a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -179,11 +179,22 @@ public class RegularContainerAllocator extends 
AbstractContainerAllocator {
 // This is to make sure non-partitioned-resource-request will prefer
 // to be allocated to non-partitioned nodes
 int missedNonPartitionedRequestSchedulingOpportunity = 0;
+AppPlacementAllocator appPlacementAllocator =
+appInfo.getAppPlacementAllocator(schedulerKey);
+if (null == appPlacementAllocator){
+  // This is possible when #pending resource decreased by a different
+  // thread.
+  ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
+  activitiesManager, node, application, priority,
+  
ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST);
+  return ContainerAllocation.PRIORITY_SKIPPED;
+}
+String requestPartition =
+appPlacementAllocator.getPrimaryRequestedNodePartition();
+
 // Only do this when request associated with given scheduler key accepts
 // NO_LABEL under RESPECT_EXCLUSIVITY mode
-if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL,
-appInfo.getAppPlacementAllocator(schedulerKey)
-.getPrimaryRequestedNodePartition())) {
+if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL, requestPartition)) {
   missedNonPartitionedRequestSchedulingOpportunity =
   application.addMissedNonPartitionedRequestSchedulingOpportunity(
   schedulerKey);
@@ -261,12 +272,9 @@ public class RegularContainerAllocator extends 
AbstractContainerAllocator {
 return result;
   }
   
-  public float getLocalityWaitFactor(
-  SchedulerRequestKey schedulerKey, int clusterNodes) {
+  public float getLocalityWaitFactor(int uniqAsks, int clusterNodes) {
 // Estimate: Required unique resources (i.e. hosts + racks)
-int requiredResources = Math.max(
-application.getAppPlacementAllocator(schedulerKey)
-.getUniqueLocationAsks() - 1, 0);
+int requiredResources = Math.max(uniqAsks - 1, 0);
 
 // waitFactor can't be more than '1' 
 // i.e. no point skipping more than clustersize opportunities
@@ -296,10 +304,16 @@ public class RegularContainerAllocator extends 
AbstractContainerAllocator {
   if (rmContext.getScheduler().getNumClusterNodes() == 0) {
 return false;
   }
+
+  int uniqLocationAsks = 0;
+  AppPlacementAllocator appPlacementAllocator =
+  application.getAppPlacementAllocator(schedulerKey);
+  if (appPlacementAllocator != null) {
+uniqLocationAsks = appPlacementAllocator.getUniqueLocationAsks();
+  }
   // If we have only ANY requests for this schedulerKey, we should not
   // delay its scheduling.
-  if (application.getAppPlacementAllocator(schedulerKey)
-  .getUniqueLocationAsks() 

[15/50] [abbrv] hadoop git commit: YARN-8122. Added component health monitoring for YARN service. Contributed by Gour Saha

[naganarasimha_gr]

YARN-8122.  Added component health monitoring for YARN service.
Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93979db8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93979db8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93979db8

Branch: refs/heads/YARN-3409
Commit: 93979db8400186d6e557c46a2a1e2622821013c1
Parents: 09c9cf1
Author: Eric Yang 
Authored: Thu Apr 26 19:17:32 2018 -0400
Committer: Eric Yang 
Committed: Thu Apr 26 19:17:32 2018 -0400

--
 .../yarn/service/component/Component.java   |  85 ++-
 .../component/instance/ComponentInstance.java   |   7 +-
 .../yarn/service/conf/YarnServiceConf.java  |  51 +++
 .../ComponentHealthThresholdMonitor.java| 151 +++
 .../yarn/service/TestYarnNativeServices.java| 108 -
 .../markdown/yarn-service/Configurations.md |   6 +-
 .../markdown/yarn-service/YarnServiceAPI.md |  35 +
 7 files changed, 434 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/93979db8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 5a85e8f..e115841 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -48,6 +48,7 @@ import 
org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
 import org.apache.hadoop.yarn.service.api.records.ServiceState;
 import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
 import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.monitor.ComponentHealthThresholdMonitor;
 import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
 import org.apache.hadoop.yarn.service.monitor.probe.Probe;
 import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
@@ -73,6 +74,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
@@ -83,10 +85,7 @@ import static 
org.apache.hadoop.yarn.service.api.ServiceApiConstants.*;
 import static org.apache.hadoop.yarn.service.component.ComponentEventType.*;
 import static org.apache.hadoop.yarn.service.component.ComponentState.*;
 import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.*;
-import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.CONTAINER_FAILURE_THRESHOLD;
-import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_CONTAINER_FAILURE_THRESHOLD;
-import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_ENABLED;
-import static 
org.apache.hadoop.yarn.service.conf.YarnServiceConf.DEFAULT_READINESS_CHECK_ENABLED_DEFAULT;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.*;
 
 public class Component implements EventHandler {
   private static final Logger LOG = LoggerFactory.getLogger(Component.class);
@@ -112,6 +111,7 @@ public class Component implements 
EventHandler {
   // The number of containers failed since last reset. This excludes preempted,
   // disk_failed containers etc. This will be reset to 0 periodically.
   public AtomicInteger currentContainerFailure = new AtomicInteger(0);
+  private boolean healthThresholdMonitorEnabled = false;
 
   private AtomicBoolean upgradeInProgress = new AtomicBoolean(false);
   private ComponentEvent upgradeEvent;
@@ -205,6 +205,7 @@ public class Component implements 
EventHandler {
 componentSpec.getConfiguration(), scheduler.getConfig());
 createNumCompInstances(component.getNumberOfContainers());
 setDesiredContainers(component.getNumberOfContainers().intValue());
+checkAndScheduleHealthThresholdMonitor();
   }
 
   

[09/50] [abbrv] hadoop git commit: HDFS-13468. Add erasure coding metrics into ReadStatistics. (Contributed by Lei (Eddy) Xu)

[naganarasimha_gr]

HDFS-13468. Add erasure coding metrics into ReadStatistics. (Contributed by Lei 
(Eddy) Xu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8e428b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8e428b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8e428b2

Branch: refs/heads/YARN-3409
Commit: a8e428b2dc0883184b43cb776d5c7196aaa3bf56
Parents: 7ac0abd
Author: Lei Xu 
Authored: Wed Apr 25 13:59:03 2018 -0700
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../hadoop/hdfs/DFSStripedInputStream.java  |  2 +
 .../org/apache/hadoop/hdfs/ReadStatistics.java  | 29 ++
 .../org/apache/hadoop/hdfs/StripeReader.java|  8 +++
 .../hdfs/client/impl/TestBlockReaderLocal.java  | 60 
 4 files changed, 99 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8e428b2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index e7d90ed..339a02c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.ReadOption;
+import org.apache.hadoop.hdfs.protocol.BlockType;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -92,6 +93,7 @@ public class DFSStripedInputStream extends DFSInputStream {
   LocatedBlocks locatedBlocks) throws IOException {
 super(dfsClient, src, verifyChecksum, locatedBlocks);
 
+this.readStatistics.setBlockType(BlockType.STRIPED);
 assert ecPolicy != null;
 this.ecPolicy = ecPolicy;
 this.cellSize = ecPolicy.getCellSize();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8e428b2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
index 59b1418..af53f0a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ReadStatistics.java
@@ -17,6 +17,8 @@
  */
 package org.apache.hadoop.hdfs;
 
+import org.apache.hadoop.hdfs.protocol.BlockType;
+
 /**
  * A utility class that maintains statistics for reading.
  */
@@ -26,6 +28,9 @@ public class ReadStatistics {
   private long totalShortCircuitBytesRead;
   private long totalZeroCopyBytesRead;
 
+  private BlockType blockType = BlockType.CONTIGUOUS;
+  private long totalEcDecodingTimeMillis;
+
   public ReadStatistics() {
 clear();
   }
@@ -75,6 +80,21 @@ public class ReadStatistics {
 return totalBytesRead - totalLocalBytesRead;
   }
 
+  /**
+   * @return block type of the input stream. If block type != CONTIGUOUS,
+   * it is reading erasure coded data.
+   */
+  public synchronized BlockType getBlockType() {
+return blockType;
+  }
+
+  /**
+   * Return the total time in milliseconds used for erasure coding decoding.
+   */
+  public synchronized long getTotalEcDecodingTimeMillis() {
+return totalEcDecodingTimeMillis;
+  }
+
   public synchronized void addRemoteBytes(long amt) {
 this.totalBytesRead += amt;
   }
@@ -97,10 +117,19 @@ public class ReadStatistics {
 this.totalZeroCopyBytesRead += amt;
   }
 
+  public synchronized void addErasureCodingDecodingTime(long millis) {
+this.totalEcDecodingTimeMillis += millis;
+  }
+
+  synchronized void setBlockType(BlockType blockType) {
+this.blockType = blockType;
+  }
+
   public synchronized void clear() {
 this.totalBytesRead = 0;
 this.totalLocalBytesRead = 0;
 this.totalShortCircuitBytesRead = 0;
 this.totalZeroCopyBytesRead = 0;
+this.totalEcDecodingTimeMillis = 0;
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8e428b2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java

[14/50] [abbrv] hadoop git commit: MAPREDUCE-7042. Killed MR job data does not move to mapreduce.jobhistory.done-dir when ATS v2 is enabled. Contributed by Rohith Sharma K S.

[naganarasimha_gr]

MAPREDUCE-7042. Killed MR job data does not move to 
mapreduce.jobhistory.done-dir when ATS v2 is enabled. Contributed by Rohith 
Sharma K S.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83e60cd2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83e60cd2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83e60cd2

Branch: refs/heads/YARN-3409
Commit: 83e60cd2db20f655e272958ef43b1b5a084ef3e3
Parents: 2a0fa50
Author: Sunil G 
Authored: Thu Apr 26 19:07:02 2018 +0530
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../jobhistory/JobHistoryEventHandler.java  | 66 +---
 .../jobhistory/TestJobHistoryEventHandler.java  | 29 +
 2 files changed, 87 insertions(+), 8 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/83e60cd2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 52c13f1..c7e69af 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -73,6 +73,7 @@ import 
org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
 import org.apache.hadoop.yarn.client.api.TimelineClient;
 import org.apache.hadoop.yarn.client.api.TimelineV2Client;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -119,7 +120,11 @@ public class JobHistoryEventHandler extends AbstractService
 
   protected BlockingQueue eventQueue =
 new LinkedBlockingQueue();
+
+  protected boolean handleTimelineEvent = false;
+  protected AsyncDispatcher atsEventDispatcher = null;
   protected Thread eventHandlingThread;
+
   private volatile boolean stopped;
   private final Object lock = new Object();
 
@@ -279,6 +284,7 @@ public class JobHistoryEventHandler extends AbstractService
   ((MRAppMaster.RunningAppContext) context).getTimelineClient();
   timelineClient.init(conf);
 }
+handleTimelineEvent = true;
 LOG.info("Timeline service is enabled; version: " +
 YarnConfiguration.getTimelineServiceVersion(conf));
   } else {
@@ -302,10 +308,23 @@ public class JobHistoryEventHandler extends 
AbstractService
   "'json' or 'binary'.  Falling back to default value '" +
   JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT + "'.");
 }
-
+// initiate the atsEventDispatcher for timeline event
+// if timeline service is enabled.
+if (handleTimelineEvent) {
+  atsEventDispatcher = createDispatcher();
+  EventHandler timelineEventHandler =
+  new ForwardingEventHandler();
+  atsEventDispatcher.register(EventType.class, timelineEventHandler);
+  atsEventDispatcher.setDrainEventsOnStop();
+  atsEventDispatcher.init(conf);
+}
 super.serviceInit(conf);
   }
 
+  protected AsyncDispatcher createDispatcher() {
+return new AsyncDispatcher("Job ATS Event Dispatcher");
+  }
+
   private void mkdir(FileSystem fs, Path path, FsPermission fsp)
   throws IOException {
 if (!fs.exists(path)) {
@@ -371,6 +390,10 @@ public class JobHistoryEventHandler extends AbstractService
 }
 }, "eventHandlingThread");
 eventHandlingThread.start();
+
+if (handleTimelineEvent) {
+  atsEventDispatcher.start();
+}
 super.serviceStart();
   }
 
@@ -461,6 +484,11 @@ public class JobHistoryEventHandler extends AbstractService
 LOG.info("Exception while closing file " + e.getMessage());
   }
 }
+
+if (handleTimelineEvent && atsEventDispatcher != null) {
+  atsEventDispatcher.stop();
+}
+
 if (timelineClient != null) {
   timelineClient.stop();
 } else if (timelineV2Client != null) {
@@ -580,6 +608,10 @@ public class JobHistoryEventHandler extends AbstractService
   }
 
   eventQueue.put(event);
+  // Process it for ATS (if enabled)

[39/50] [abbrv] hadoop git commit: YARN-7875. Node Attribute store for storing and recovering attributes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-7875. Node Attribute store for storing and recovering attributes. 
Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5ff4fac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5ff4fac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5ff4fac

Branch: refs/heads/YARN-3409
Commit: f5ff4fac156423e37028d9f4d17cf10d632304a8
Parents: a9fb829
Author: Sunil G 
Authored: Fri Apr 6 07:09:27 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  16 ++
 .../yarn/nodelabels/NodeAttributeStore.java |  77 ++
 .../yarn/nodelabels/NodeAttributesManager.java  |  11 +
 .../hadoop/yarn/nodelabels/RMNodeAttribute.java |   6 -
 .../nodelabels/store/AbstractFSNodeStore.java   |   2 +-
 .../yarn/nodelabels/store/FSStoreOpHandler.java |  21 +-
 .../store/op/AddNodeToAttributeLogOp.java   |  71 +
 .../nodelabels/store/op/FSNodeStoreLogOp.java   |  17 ++
 .../store/op/NodeAttributeMirrorOp.java |  64 +
 .../store/op/RemoveNodeToAttributeLogOp.java|  71 +
 .../store/op/ReplaceNodeToAttributeLogOp.java   |  73 ++
 .../yarn/nodelabels/store/op/package-info.java  |  21 ++
 .../src/main/resources/yarn-default.xml |  16 ++
 .../FileSystemNodeAttributeStore.java   | 102 
 .../nodelabels/NodeAttributesManagerImpl.java   | 100 ++-
 .../TestResourceTrackerService.java |  10 +
 .../TestFileSystemNodeAttributeStore.java   | 260 +++
 .../nodelabels/TestNodeAttributesManager.java   |  13 +-
 18 files changed, 935 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5ff4fac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index fa61ef3..4e8886d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3402,6 +3402,22 @@ public class YarnConfiguration extends Configuration {
   + "fs-store.root-dir";
 
   /**
+   * Node-attribute configurations.
+   */
+  public static final String NODE_ATTRIBUTE_PREFIX =
+  YARN_PREFIX + "node-attribute.";
+  /**
+   * Node attribute store implementation class.
+   */
+  public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS =
+  NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class";
+  /**
+   * File system not attribute store directory.
+   */
+  public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR =
+  NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir";
+
+  /**
* Flag to indicate if the node labels feature enabled, by default it's
* disabled
*/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5ff4fac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
new file mode 100644
index 000..8e9f9ff
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributeStore.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 

[40/50] [abbrv] hadoop git commit: YARN-7871. Node attributes reporting from NM to RM. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7871. Node attributes reporting from NM to RM. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4112d70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4112d70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4112d70

Branch: refs/heads/YARN-3409
Commit: d4112d70d90b7392c902986b2bae1457ba99b5fc
Parents: 9553153
Author: Naganarasimha 
Authored: Mon Mar 12 08:05:53 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  10 +-
 .../yarn/nodelabels/NodeAttributesManager.java  |  17 +-
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   |  19 ++
 .../src/main/resources/yarn-default.xml |  24 +++
 .../yarn/server/nodemanager/NodeManager.java|  70 +--
 .../server/nodemanager/NodeStatusUpdater.java   |  14 ++
 .../nodemanager/NodeStatusUpdaterImpl.java  |  70 ++-
 .../ConfigurationNodeAttributesProvider.java|  90 +
 .../server/nodemanager/TestNodeManager.java |   2 +-
 .../TestNodeStatusUpdaterForLabels.java |  10 +-
 ...TestConfigurationNodeAttributesProvider.java | 185 +++
 .../resourcemanager/ResourceTrackerService.java |  30 +++
 .../nodelabels/NodeAttributesManagerImpl.java   |  52 --
 .../TestResourceTrackerService.java |  78 
 .../nodelabels/TestNodeAttributesManager.java   |  99 ++
 15 files changed, 718 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4112d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b908faa..fa61ef3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3469,9 +3469,12 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_NODE_LABELS_PROVIDER_CONFIG =
   NM_NODE_LABELS_PREFIX + "provider";
 
+  public static final String NM_NODE_ATTRIBUTES_PROVIDER_CONFIG =
+  NM_NODE_ATTRIBUTES_PREFIX + "provider";
+
   // whitelist names for the yarn.nodemanager.node-labels.provider
-  public static final String CONFIG_NODE_LABELS_PROVIDER = "config";
-  public static final String SCRIPT_NODE_LABELS_PROVIDER = "script";
+  public static final String CONFIG_NODE_DESCRIPTOR_PROVIDER = "config";
+  public static final String SCRIPT_NODE_DESCRIPTOR_PROVIDER = "script";
 
   private static final String NM_NODE_LABELS_PROVIDER_PREFIX =
   NM_NODE_LABELS_PREFIX + "provider.";
@@ -3503,6 +3506,9 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_PROVIDER_CONFIGURED_NODE_PARTITION =
   NM_NODE_LABELS_PROVIDER_PREFIX + "configured-node-partition";
 
+  public static final String NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES =
+  NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "configured-node-attributes";
+
   private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX
   + "node-labels.";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4112d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
index effda9b..ffa33cf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -35,15 +35,18 @@ public abstract class NodeAttributesManager extends 
AbstractService {
 
   /**
* To completely replace the mappings for a given node with the new Set of
-   * Attributes. If the mapping contains an attribute whose type does not match
-   * a previously existing Attribute under the same prefix (name space) then
-   * exception is thrown. Key would be name of the node and value would be set
-   * of Attributes 

[22/50] [abbrv] hadoop git commit: HADOOP-14188. Remove the usage of org.mockito.internal.util.reflection.Whitebox

[naganarasimha_gr]

HADOOP-14188. Remove the usage of org.mockito.internal.util.reflection.Whitebox

Signed-off-by: Akira Ajisaka 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84ecfe3c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84ecfe3c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84ecfe3c

Branch: refs/heads/YARN-3409
Commit: 84ecfe3cebe32ae048eae4b1bf02f6cdfd1fafe9
Parents: ba2db39
Author: Ewan Higgs 
Authored: Fri Apr 27 19:21:37 2018 +0900
Committer: Akira Ajisaka 
Committed: Fri Apr 27 19:21:37 2018 +0900

--
 ...yptoStreamsWithOpensslAesCtrCryptoCodec.java |   2 +-
 .../apache/hadoop/fs/TestLocalFileSystem.java   |   2 +-
 .../org/apache/hadoop/http/TestHttpServer.java  |   2 +-
 .../java/org/apache/hadoop/ipc/TestIPC.java |   2 +-
 .../java/org/apache/hadoop/ipc/TestRPC.java |   2 +-
 .../metrics2/impl/TestGraphiteMetrics.java  |   2 +-
 .../hadoop/metrics2/impl/TestStatsDMetrics.java |   2 +-
 .../java/org/apache/hadoop/test/Whitebox.java   | 113 +++
 .../hadoop/crypto/key/kms/server/TestKMS.java   |   2 +-
 .../crypto/key/kms/server/TestKMSAudit.java |   2 +-
 .../org/apache/hadoop/portmap/TestPortmap.java  |   2 +-
 .../hdfs/web/TestByteRangeInputStream.java  |   2 +-
 .../apache/hadoop/hdfs/web/TestTokenAspect.java |   2 +-
 .../server/federation/FederationTestUtils.java  |   2 +-
 .../router/TestRouterRPCClientRetries.java  |   2 +-
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |   2 +-
 .../apache/hadoop/hdfs/TestDFSOutputStream.java |   2 +-
 .../hadoop/hdfs/TestDistributedFileSystem.java  |   2 +-
 .../hadoop/hdfs/TestEncryptionZonesWithKMS.java |   2 +-
 .../hadoop/hdfs/TestLeaseRecoveryStriped.java   |   2 +-
 .../TestDelegationTokenForProxyUser.java|   2 +-
 .../hdfs/server/balancer/TestKeyManager.java|   2 +-
 .../blockmanagement/BlockManagerTestUtil.java   |   2 +-
 .../blockmanagement/TestBlockInfoStriped.java   |   2 +-
 .../TestBlockManagerSafeMode.java   |   3 +-
 .../TestComputeInvalidateWork.java  |   2 +-
 .../blockmanagement/TestDatanodeManager.java|   2 +-
 .../blockmanagement/TestHostFileManager.java|   3 +-
 .../TestPendingInvalidateBlock.java |   3 +-
 ...constructStripedBlocksWithRackAwareness.java |   2 +-
 .../TestSequentialBlockGroupId.java |   2 +-
 .../TestUnderReplicatedBlocks.java  |   2 +-
 .../server/datanode/TestBlockPoolManager.java   |   2 +-
 .../hdfs/server/namenode/NameNodeAdapter.java   |   2 +-
 .../namenode/TestAddStripedBlockInFBR.java  |   2 +-
 .../TestCommitBlockSynchronization.java |   2 +-
 .../hdfs/server/namenode/TestDeleteRace.java|   2 +-
 .../hdfs/server/namenode/TestFSNamesystem.java  |   2 +-
 .../hdfs/server/namenode/TestReencryption.java  |   2 +-
 .../namenode/TestReencryptionHandler.java   |   2 +-
 .../hdfs/server/namenode/TestSaveNamespace.java |   2 +-
 .../namenode/TestTruncateQuotaUpdate.java   |   2 +-
 .../ha/TestBootstrapStandbyWithQJM.java |   2 +-
 .../namenode/ha/TestDFSUpgradeWithHA.java   |   2 +-
 .../namenode/ha/TestDelegationTokensWithHA.java |   2 +-
 .../hdfs/server/namenode/ha/TestHASafeMode.java |   2 +-
 .../snapshot/TestFileWithSnapshotFeature.java   |   2 +-
 .../snapshot/TestRenameWithSnapshots.java   |   2 +-
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java |   2 +-
 .../hadoop/hdfs/web/TestWebHDFSForHA.java   |   2 +-
 .../hadoop/hdfs/web/TestWebHdfsTokens.java  |   2 +-
 .../mapreduce/task/reduce/TestMergeManager.java |   2 +-
 .../apache/hadoop/ozone/ksm/TestKSMMetrcis.java |   2 +-
 .../launcher/TestContainersLauncher.java|   2 +-
 54 files changed, 167 insertions(+), 55 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/84ecfe3c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index 241e876..6b56aaf 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -21,9 +21,9 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.random.OsSecureRandom;
 import 

[08/50] [abbrv] hadoop git commit: HADOOP-15411. AuthenticationFilter should use Configuration.getPropsWithPrefix instead of iterator. (Suma Shivaprasad via wangda)

[naganarasimha_gr]

HADOOP-15411. AuthenticationFilter should use Configuration.getPropsWithPrefix 
instead of iterator. (Suma Shivaprasad via wangda)

Change-Id: Ia19458a43aceb4ffdb1e8eccc519655235d92ba9


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13d389bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13d389bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13d389bf

Branch: refs/heads/YARN-3409
Commit: 13d389bf5125c9f3bbc6863d93a6371fc2930417
Parents: a8e428b
Author: Wangda Tan 
Authored: Wed Apr 25 22:09:37 2018 -0700
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../hadoop/security/AuthenticationFilterInitializer.java  | 10 +++---
 1 file changed, 3 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d389bf/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
index ca221f5..7e6b3a8 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
@@ -69,14 +69,10 @@ public class AuthenticationFilterInitializer extends 
FilterInitializer {
 
 //setting the cookie path to root '/' so it is used for all resources.
 filterConfig.put(AuthenticationFilter.COOKIE_PATH, "/");
+Map propsWithPrefix = conf.getPropsWithPrefix(prefix);
 
-for (Map.Entry entry : conf) {
-  String name = entry.getKey();
-  if (name.startsWith(prefix)) {
-String value = conf.get(name);
-name = name.substring(prefix.length());
-filterConfig.put(name, value);
-  }
+for (Map.Entry entry : propsWithPrefix.entrySet()) {
+  filterConfig.put(entry.getKey(), entry.getValue());
 }
 
 //Resolve _HOST into bind address


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[26/50] [abbrv] hadoop git commit: YARN-7781. Update YARN service documentation. Contributed by Gour Saha

[naganarasimha_gr]

YARN-7781. Update YARN service documentation.
   Contributed by Gour Saha


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24a5ccbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24a5ccbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24a5ccbf

Branch: refs/heads/YARN-3409
Commit: 24a5ccbf4bda413a98480d52c204d56f82ef9ac5
Parents: 14b4799
Author: Eric Yang 
Authored: Fri Apr 27 12:38:30 2018 -0400
Committer: Eric Yang 
Committed: Fri Apr 27 12:38:30 2018 -0400

--
 .../definition/YARN-Services-Examples.md| 236 +--
 ...RN-Simplified-V1-API-Layer-For-Services.yaml |  21 +-
 .../yarn/service/api/records/BaseResource.java  |   2 +-
 .../site/markdown/yarn-service/QuickStart.md|  13 +-
 .../markdown/yarn-service/YarnServiceAPI.md |  61 +++--
 5 files changed, 279 insertions(+), 54 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/24a5ccbf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
index 22f941e..a4ef2d2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/resources/definition/YARN-Services-Examples.md
@@ -15,7 +15,7 @@
 ## Examples
 
 ### Create a simple single-component service with most attribute values as 
defaults
-POST URL - http://localhost:9191/ws/v1/services
+POST URL - http://localhost:8088/app/v1/services
 
 # POST Request JSON
 ```json
@@ -27,7 +27,7 @@ POST URL - http://localhost:9191/ws/v1/services
 [
   {
 "name": "hello",
-"number_of_containers": 1,
+"number_of_containers": 2,
 "artifact": {
   "id": "nginx:latest",
   "type": "DOCKER"
@@ -36,14 +36,14 @@ POST URL - http://localhost:9191/ws/v1/services
 "resource": {
   "cpus": 1,
   "memory": "256"
-   }
+}
   }
 ]
 }
 ```
 
 # GET Response JSON
-GET URL - http://localhost:9191/ws/v1/services/hello-world
+GET URL - http://localhost:8088/app/v1/services/hello-world
 
 Note, lifetime value of -1 means unlimited lifetime.
 
@@ -54,10 +54,11 @@ Note, lifetime value of -1 means unlimited lifetime.
 "description": "hello world example",
 "id": "application_1503963985568_0002",
 "lifetime": -1,
+"state": "STABLE",
 "components": [
 {
 "name": "hello",
-"dependencies": [],
+"state": "STABLE",
 "resource": {
 "cpus": 1,
 "memory": "256"
@@ -70,21 +71,21 @@ Note, lifetime value of -1 means unlimited lifetime.
 "quicklinks": [],
 "containers": [
 {
-"id": "container_e03_1503963985568_0002_01_01",
+"id": "container_e03_1503963985568_0002_01_02",
 "ip": "10.22.8.143",
-"hostname": "myhost.local",
+"hostname": 
"ctr-e03-1503963985568-0002-01-02.example.site",
 "state": "READY",
 "launch_time": 1504051512412,
-"bare_host": "10.22.8.143",
+"bare_host": "host100.cloud.com",
 "component_instance_name": "hello-0"
 },
 {
-"id": "container_e03_1503963985568_0002_01_02",
-"ip": "10.22.8.143",
-"hostname": "myhost.local",
+"id": "container_e03_1503963985568_0002_01_03",
+"ip": "10.22.8.144",
+"hostname": 
"ctr-e03-1503963985568-0002-01-03.example.site",
 "state": "READY",
 "launch_time": 1504051536450,
-"bare_host": "10.22.8.143",
+"bare_host": "host100.cloud.com",
 "component_instance_name": "hello-1"
 }
 ],
@@ -103,7 +104,7 @@ Note, lifetime value of -1 means unlimited lifetime.
 
 ```
 ### Update to modify the lifetime of a service
-PUT URL - http://localhost:9191/ws/v1/services/hello-world
+PUT URL - http://localhost:8088/app/v1/services/hello-world
 
 

[38/50] [abbrv] hadoop git commit: YARN-8104. Add API to fetch node to attribute mapping. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-8104. Add API to fetch node to attribute mapping. Contributed by Bibin A 
Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/011795f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/011795f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/011795f7

Branch: refs/heads/YARN-3409
Commit: 011795f7462920daeb6887184fb994524ceae838
Parents: 13b0c11
Author: Naganarasimha 
Authored: Fri Apr 20 07:31:53 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/mapred/ResourceMgrDelegate.java  |   6 +
 .../hadoop/mapred/TestClientRedirect.java   |   8 +
 .../yarn/api/ApplicationClientProtocol.java |  18 ++
 .../GetNodesToAttributesRequest.java|  65 +++
 .../GetNodesToAttributesResponse.java   |  63 +++
 .../main/proto/applicationclient_protocol.proto |   1 +
 ..._server_resourcemanager_service_protos.proto |   5 -
 .../src/main/proto/yarn_protos.proto|   5 +
 .../src/main/proto/yarn_service_protos.proto|  10 +-
 .../hadoop/yarn/client/api/YarnClient.java  |  19 ++
 .../yarn/client/api/impl/YarnClientImpl.java|   9 +
 .../ApplicationClientProtocolPBClientImpl.java  |  18 ++
 .../ApplicationClientProtocolPBServiceImpl.java |  21 +++
 .../pb/GetAttributesToNodesResponsePBImpl.java  |   6 +-
 .../pb/GetNodesToAttributesRequestPBImpl.java   | 132 ++
 .../pb/GetNodesToAttributesResponsePBImpl.java  | 181 +++
 .../yarn/nodelabels/NodeAttributesManager.java  |   9 +
 .../impl/pb/NodeToAttributesPBImpl.java |   4 +-
 .../NodesToAttributesMappingRequestPBImpl.java  |   2 +-
 .../hadoop/yarn/api/TestPBImplRecords.java  |  16 +-
 .../yarn/server/MockResourceManagerFacade.java  |   8 +
 .../server/resourcemanager/ClientRMService.java |  13 ++
 .../nodelabels/NodeAttributesManagerImpl.java   |  24 +++
 .../resourcemanager/TestClientRMService.java|  86 +
 .../DefaultClientRequestInterceptor.java|   8 +
 .../clientrm/FederationClientInterceptor.java   |   8 +
 .../router/clientrm/RouterClientRMService.java  |   9 +
 .../PassThroughClientRequestInterceptor.java|   8 +
 28 files changed, 749 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/011795f7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 4f96a6b..1a7f308 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -551,4 +551,10 @@ public class ResourceMgrDelegate extends YarnClient {
   Set attributes) throws YarnException, IOException {
 return client.getAttributesToNodes(attributes);
   }
+
+  @Override
+  public Map getNodeToAttributes(
+  Set hostNames) throws YarnException, IOException {
+return client.getNodeToAttributes(hostNames);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/011795f7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index 23a1a85..5972f65 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
@@ -104,6 +104,8 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest;
 import 

[32/50] [abbrv] hadoop git commit: HDFS-13508. RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries. Contributed by Ekanth S.

[naganarasimha_gr]

HDFS-13508. RBF: Normalize paths (automatically) when adding, updating, 
removing or listing mount table entries. Contributed by Ekanth S.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48444060
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48444060
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48444060

Branch: refs/heads/YARN-3409
Commit: 484440602c5b69fbd8106010603c61ae051056dd
Parents: f469628
Author: Inigo Goiri 
Authored: Fri Apr 27 16:28:17 2018 -0700
Committer: Inigo Goiri 
Committed: Fri Apr 27 16:28:17 2018 -0700

--
 .../hdfs/tools/federation/RouterAdmin.java  |  16 +++
 .../federation/router/TestRouterAdminCLI.java   | 117 ++-
 2 files changed, 130 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/48444060/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
index 17707dc..b0a2062 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -322,6 +323,7 @@ public class RouterAdmin extends Configured implements Tool 
{
   public boolean addMount(String mount, String[] nss, String dest,
   boolean readonly, DestinationOrder order, ACLEntity aclInfo)
   throws IOException {
+mount = normalizeFileSystemPath(mount);
 // Get the existing entry
 MountTableManager mountTable = client.getMountTableManager();
 GetMountTableEntriesRequest getRequest =
@@ -473,6 +475,7 @@ public class RouterAdmin extends Configured implements Tool 
{
   public boolean updateMount(String mount, String[] nss, String dest,
   boolean readonly, DestinationOrder order, ACLEntity aclInfo)
   throws IOException {
+mount = normalizeFileSystemPath(mount);
 MountTableManager mountTable = client.getMountTableManager();
 
 // Create a new entry
@@ -519,6 +522,7 @@ public class RouterAdmin extends Configured implements Tool 
{
* @throws IOException If it cannot be removed.
*/
   public boolean removeMount(String path) throws IOException {
+path = normalizeFileSystemPath(path);
 MountTableManager mountTable = client.getMountTableManager();
 RemoveMountTableEntryRequest request =
 RemoveMountTableEntryRequest.newInstance(path);
@@ -538,6 +542,7 @@ public class RouterAdmin extends Configured implements Tool 
{
* @throws IOException If it cannot be listed.
*/
   public void listMounts(String path) throws IOException {
+path = normalizeFileSystemPath(path);
 MountTableManager mountTable = client.getMountTableManager();
 GetMountTableEntriesRequest request =
 GetMountTableEntriesRequest.newInstance(path);
@@ -798,6 +803,17 @@ public class RouterAdmin extends Configured implements 
Tool {
   }
 
   /**
+   * Normalize a path for that filesystem.
+   *
+   * @param path Path to normalize.
+   * @return Normalized path.
+   */
+  private static String normalizeFileSystemPath(final String path) {
+Path normalizedPath = new Path(path);
+return normalizedPath.toString();
+  }
+
+  /**
* Inner class that stores ACL info of mount table.
*/
   static class ACLEntity {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/48444060/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
index 4e84c33..2537c19 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
+++ 

[13/50] [abbrv] hadoop git commit: YARN-7939. Added support to upgrade a component instance. Contributed by Chandni Singh

[naganarasimha_gr]

YARN-7939.  Added support to upgrade a component instance.
Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a7369b0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a7369b0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a7369b0

Branch: refs/heads/YARN-3409
Commit: 4a7369b09547a78f532af66c763c6994a38b5d68
Parents: 83e60cd
Author: Eric Yang 
Authored: Thu Apr 26 15:47:55 2018 -0400
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../yarn/service/client/ApiServiceClient.java   |  86 --
 .../hadoop/yarn/service/webapp/ApiServer.java   | 188 +++--
 .../hadoop/yarn/service/ServiceClientTest.java  | 120 +++--
 .../hadoop/yarn/service/TestApiServer.java  |  87 --
 .../service/client/TestApiServiceClient.java|  26 ++
 .../hadoop/yarn/service/ClientAMProtocol.java   |   6 +
 .../hadoop/yarn/service/ClientAMService.java|  38 ++-
 .../hadoop/yarn/service/ServiceContext.java |  10 +
 .../hadoop/yarn/service/ServiceEvent.java   |  10 +
 .../hadoop/yarn/service/ServiceEventType.java   |   2 +-
 .../hadoop/yarn/service/ServiceManager.java | 156 ---
 .../hadoop/yarn/service/ServiceScheduler.java   |  27 ++
 .../yarn/service/api/records/Component.java |  21 ++
 .../service/api/records/ContainerState.java |   2 +-
 .../yarn/service/api/records/ServiceState.java  |   3 +-
 .../yarn/service/client/ServiceClient.java  |  87 +-
 .../yarn/service/component/Component.java   | 153 +--
 .../yarn/service/component/ComponentEvent.java  |  10 +
 .../service/component/ComponentEventType.java   |   2 +-
 .../component/instance/ComponentInstance.java   |  50 +++-
 .../instance/ComponentInstanceEventType.java|   3 +-
 .../yarn/service/conf/RestApiConstants.java |  12 +
 .../containerlaunch/ContainerLaunchService.java | 100 ++-
 .../pb/client/ClientAMProtocolPBClientImpl.java |  14 +
 .../service/ClientAMProtocolPBServiceImpl.java  |  12 +
 .../provider/AbstractProviderService.java   |  25 +-
 .../yarn/service/provider/ProviderService.java  |   7 +-
 .../yarn/service/provider/ProviderUtils.java|  27 +-
 .../yarn/service/utils/ServiceApiUtil.java  |  48 +++-
 .../src/main/proto/ClientAMProtocol.proto   |  11 +
 .../hadoop/yarn/service/TestServiceAM.java  |   5 +
 .../hadoop/yarn/service/TestServiceManager.java | 122 -
 .../yarn/service/TestYarnNativeServices.java|  34 ++-
 .../yarn/service/client/TestServiceCLI.java |  90 +--
 .../yarn/service/client/TestServiceClient.java  | 187 +
 .../yarn/service/component/TestComponent.java   | 265 +++
 .../instance/TestComponentInstance.java |  88 ++
 .../service/monitor/TestServiceMonitor.java |   1 +
 .../hadoop/yarn/client/api/AppAdminClient.java  |  27 +-
 .../hadoop/yarn/client/cli/ApplicationCLI.java  |  65 +
 .../hadoop/yarn/client/cli/TestYarnCLI.java |  21 ++
 41 files changed, 1961 insertions(+), 287 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a7369b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index e4a245d..cdba555 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -26,6 +26,7 @@ import java.util.Map;
 
 import javax.ws.rs.core.MediaType;
 
+import com.google.common.base.Preconditions;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
@@ -40,11 +41,16 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.ContainerState;
 import org.apache.hadoop.yarn.service.api.records.Service;
 import 

[23/50] [abbrv] hadoop git commit: YARN-8208. Add log statement for Docker client configuration file at INFO level. Contributed by Yesha Vora

[naganarasimha_gr]

YARN-8208. Add log statement for Docker client configuration file at INFO 
level. Contributed by Yesha Vora


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/914d6b5c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/914d6b5c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/914d6b5c

Branch: refs/heads/YARN-3409
Commit: 914d6b5cb143f4f9cadd9d689ea4e278383a6779
Parents: 84ecfe3
Author: Billie Rinaldi 
Authored: Fri Apr 27 06:10:32 2018 -0700
Committer: Billie Rinaldi 
Committed: Fri Apr 27 06:10:32 2018 -0700

--
 .../org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java   | 1 +
 .../org/apache/hadoop/yarn/util/DockerClientConfigHandler.java | 6 ++
 2 files changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/914d6b5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 33919ba..a4e5c0d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -232,6 +232,7 @@ public class ServiceApiUtil {
 if (!StringUtils.isEmpty(dockerClientConfig)) {
   Path dockerClientConfigPath = new Path(dockerClientConfig);
   FileSystem fs = dockerClientConfigPath.getFileSystem(conf);
+  LOG.info("The supplied Docker client config is " + dockerClientConfig);
   if (!fs.exists(dockerClientConfigPath)) {
 throw new IOException(
 "The supplied Docker client config does not exist: "

http://git-wip-us.apache.org/repos/asf/hadoop/blob/914d6b5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
index 98be93b..5522cf4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/DockerClientConfigHandler.java
@@ -118,10 +118,8 @@ public final class DockerClientConfigHandler {
 tokenId.getKind(), new Text(registryUrl));
 credentials.addToken(
 new Text(registryUrl + "-" + applicationId), token);
-if (LOG.isDebugEnabled()) {
-  LOG.debug("Token read from Docker client configuration file: "
-  + token.toString());
-}
+LOG.info("Token read from Docker client configuration file: "
++ token.toString());
   }
 }
 return credentials;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[01/50] [abbrv] hadoop git commit: YARN-8196. Updated documentation for enabling YARN Service REST API. Contributed by Billie Rinaldi [Forced Update!]

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 341094ca8 -> 011795f74 (forced update)


YARN-8196.  Updated documentation for enabling YARN Service REST API.
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b1dad91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b1dad91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b1dad91

Branch: refs/heads/YARN-3409
Commit: 3b1dad91232ef24e3cb01d7752b11feb1d80262c
Parents: 2adda92
Author: Eric Yang 
Authored: Tue Apr 24 19:11:21 2018 -0400
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../site/markdown/yarn-service/QuickStart.md| 30 +---
 1 file changed, 14 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b1dad91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index f563193..e91380c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -18,10 +18,21 @@ This document describes how to deploy services on YARN 
using the YARN Service fr
 
 
 
-## Start HDFS and YARN components
+## Configure and start HDFS and YARN components
 
- Start all the hadoop components HDFS, YARN as usual.
+Start all the hadoop components for HDFS and YARN as usual.
+To enable the YARN Service framework, add this property to `yarn-site.xml` and 
restart the ResourceManager or set the property before the ResourceManager is 
started.
+This property is required for using the YARN Service framework through the CLI 
or the REST API.
 
+```
+  
+
+  Enable services rest api on ResourceManager.
+
+yarn.webapp.api-service.enable
+true
+  
+```
 
 ## Example service 
 Below is a simple service definition that launches sleep containers on YARN by 
writing a simple spec file and without writing any code.
@@ -104,20 +115,7 @@ yarn app -destroy ${SERVICE_NAME}
 
 ## Manage services on YARN via REST API
 
-YARN API Server REST API can be activated as part of the ResourceManager.
-
-### Start Embedded API-Server as part of ResourceManager
-For running inside ResourceManager, add this property to `yarn-site.xml` and 
restart ResourceManager.
-
-```
-  
-
-  Enable services rest api on ResourceManager.
-
-yarn.webapp.api-service.enable
-true
-  
-```
+The YARN API Server REST API is activated as part of the ResourceManager when 
`yarn.webapp.api-service.enable` is set to true.
 
 Services can be deployed on YARN through the ResourceManager web endpoint.
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[04/50] [abbrv] hadoop git commit: HADOOP-15402. Prevent double logout of UGI's LoginContext

[naganarasimha_gr]

HADOOP-15402. Prevent double logout of UGI's LoginContext

Signed-off-by: Akira Ajisaka 


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbc8dfcb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbc8dfcb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbc8dfcb

Branch: refs/heads/YARN-3409
Commit: bbc8dfcb8b5acd92aa3a2d945a1d51403cdde874
Parents: 6f655ae
Author: Daryn Sharp 
Authored: Wed Apr 25 17:05:42 2018 +0900
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../java/org/apache/hadoop/security/UserGroupInformation.java  | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbc8dfcb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index d0522a0..a9f6cb6 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -47,6 +47,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 
 import javax.security.auth.DestroyFailedException;
@@ -1843,6 +1844,7 @@ public class UserGroupInformation {
   private static class HadoopLoginContext extends LoginContext {
 private final String appName;
 private final HadoopConfiguration conf;
+private AtomicBoolean isLoggedIn = new AtomicBoolean();
 
 HadoopLoginContext(String appName, Subject subject,
HadoopConfiguration conf) throws LoginException {
@@ -1875,6 +1877,7 @@ public class UserGroupInformation {
 long start = Time.monotonicNow();
 try {
   super.login();
+  isLoggedIn.set(true);
   metric = metrics.loginSuccess;
 } finally {
   metric.add(Time.monotonicNow() - start);
@@ -1885,8 +1888,7 @@ public class UserGroupInformation {
 @Override
 public void logout() throws LoginException {
   synchronized(getSubjectLock()) {
-if (this.getSubject() != null
-&& !this.getSubject().getPrivateCredentials().isEmpty()) {
+if (isLoggedIn.compareAndSet(true, false)) {
   super.logout();
 }
   }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[24/50] [abbrv] hadoop git commit: YARN-8211. Yarn registry dns log finds BufferUnderflowException on port ping. Contributed by Eric Yang

[naganarasimha_gr]

YARN-8211. Yarn registry dns log finds BufferUnderflowException on port ping. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ab3f970
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ab3f970
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ab3f970

Branch: refs/heads/YARN-3409
Commit: 9ab3f9708923b3429209006a12695f4826a045b9
Parents: 914d6b5
Author: Billie Rinaldi 
Authored: Fri Apr 27 07:58:43 2018 -0700
Committer: Billie Rinaldi 
Committed: Fri Apr 27 07:58:43 2018 -0700

--
 .../java/org/apache/hadoop/registry/server/dns/RegistryDNS.java   | 3 +++
 1 file changed, 3 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ab3f970/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
index c31ce89..c322023 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/RegistryDNS.java
@@ -76,6 +76,7 @@ import java.net.Socket;
 import java.net.SocketAddress;
 import java.net.SocketException;
 import java.net.UnknownHostException;
+import java.nio.BufferUnderflowException;
 import java.nio.ByteBuffer;
 import java.nio.channels.DatagramChannel;
 import java.nio.channels.ServerSocketChannel;
@@ -802,6 +803,8 @@ public class RegistryDNS extends AbstractService implements 
DNSOperations,
   ch.socket().getPort(),
   ch.socket().getLocalAddress().getHostName(),
   ch.socket().getLocalPort(), e);
+} catch (BufferUnderflowException e) {
+  // Ignore system monitor ping packets
 } finally {
   IOUtils.closeStream(ch);
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[10/50] [abbrv] hadoop git commit: YARN-8188. RM Nodes UI data table index for sorting column need to be corrected post Application tags display. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-8188. RM Nodes UI data table index for sorting column need to be corrected 
post Application tags display. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ac0abdc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ac0abdc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ac0abdc

Branch: refs/heads/YARN-3409
Commit: 7ac0abdc13abce6dd95643c48d84254ac9fcec78
Parents: bbc8dfc
Author: Sunil G 
Authored: Wed Apr 25 14:57:16 2018 +0530
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../hadoop/yarn/server/resourcemanager/webapp/NodesPage.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ac0abdc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
index 3e78cf4..09b1c0f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
@@ -235,7 +235,7 @@ class NodesPage extends RmView {
 .append(", aoColumnDefs: [");
 b.append("{'bSearchable': false, 'aTargets': [ 7 ]}");
 b.append(", {'sType': 'title-numeric', 'bSearchable': false, "
-+ "'aTargets': [ 8, 9 ] }");
++ "'aTargets': [ 9, 10 ] }");
 b.append(", {'sType': 'title-numeric', 'aTargets': [ 5 ]}");
 b.append("]}");
 return b.toString();


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[16/50] [abbrv] hadoop git commit: HDFS-13499. RBF: Show disabled name services in the UI. Contributed by Inigo Goiri.

[naganarasimha_gr]

HDFS-13499. RBF: Show disabled name services in the UI. Contributed by Inigo 
Goiri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a293de0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a293de0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a293de0

Branch: refs/heads/YARN-3409
Commit: 0a293de00d5f123c7c8248459aa8ae88a45f39b1
Parents: 93979db
Author: Yiqun Lin 
Authored: Fri Apr 27 10:38:15 2018 +0800
Committer: Yiqun Lin 
Committed: Fri Apr 27 10:38:15 2018 +0800

--
 .../src/main/webapps/router/federationhealth.html   | 1 +
 .../hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js | 3 +++
 .../hadoop-hdfs-rbf/src/main/webapps/static/rbf.css | 5 +
 3 files changed, 9 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a293de0/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
index f1cf482..37fcb92 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.html
@@ -132,6 +132,7 @@
 Active
 Standby
 Safe mode
+Disabled
 Unavailable
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a293de0/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
index a0b0128..6779b61 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/router/federationhealth.js
@@ -130,6 +130,9 @@
   } else if (nodes[i].state === "UNAVAILABLE") {
 n.title = capitalise(n.state);
 n.iconState = "unavailable";
+  } else if (nodes[i].state === "DISABLED") {
+n.title = capitalise(n.state);
+n.iconState = "disabled";
   }
   if (n.namenodeId === "null") {
 n.namenodeId = "";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a293de0/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/static/rbf.css
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/static/rbf.css 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/static/rbf.css
index f923085..43112af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/static/rbf.css
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps/static/rbf.css
@@ -53,6 +53,11 @@
 content: "\e090";
 }
 
+.federationhealth-namenode-disabled:before {
+color: #eea236;
+content: "\e136";
+}
+
 .federationhealth-namenode-legend {
 list-style-type: none;
 text-align: right;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[44/50] [abbrv] hadoop git commit: YARN-8117. Fix TestRMWebServicesNodes test failure. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-8117. Fix TestRMWebServicesNodes test failure. Contributed by Bibin A 
Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9fb8294
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9fb8294
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9fb8294

Branch: refs/heads/YARN-3409
Commit: a9fb8294e96e64097585ad5adcedc831c8d9a4cb
Parents: 0346915
Author: bibinchundatt 
Authored: Thu Apr 5 14:09:50 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9fb8294/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 7ea7e81..72b7a45 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -740,7 +740,7 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
 
   public void verifyNodeInfo(JSONObject nodeInfo, RMNode nm)
   throws JSONException, Exception {
-assertEquals("incorrect number of elements", 19, nodeInfo.length());
+assertEquals("incorrect number of elements", 20, nodeInfo.length());
 
 JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization");
 verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[43/50] [abbrv] hadoop git commit: YARN-7965. NodeAttributeManager add/get API is not working properly. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7965. NodeAttributeManager add/get API is not working properly. 
Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/95531532
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/95531532
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/95531532

Branch: refs/heads/YARN-3409
Commit: 9553153265c4b04fdc95bcafeec443bc3e854566
Parents: ebb4b6e
Author: Naganarasimha 
Authored: Tue Feb 27 18:46:16 2018 +0800
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../yarn/nodelabels/NodeAttributesManager.java  |   8 +-
 .../nodelabels/NodeAttributesManagerImpl.java   |  38 ++-
 .../nodelabels/TestNodeAttributesManager.java   | 258 +++
 3 files changed, 288 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/95531532/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
index 63f3dcf..effda9b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -69,10 +69,14 @@ public abstract class NodeAttributesManager extends 
AbstractService {
   Map nodeAttributeMapping) throws IOException;
 
   /**
+   * Returns a set of node attributes whose prefix is one of the given
+   * prefixes; if the prefix set is null or empty, all attributes are returned;
+   * if prefix set is given but no mapping could be found, an empty set
+   * is returned.
+   *
* @param prefix set of prefix string's for which the attributes needs to
*  returned
-   * @return set of node Attributes objects for the specified set of prefixes,
-   * else return all
+   * @return set of node Attributes
*/
   public abstract Set getClusterNodeAttributes(
   Set prefix);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/95531532/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index 2e63a7c..a902ac6 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -27,6 +27,7 @@ import java.util.Map.Entry;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentHashMap.KeySetView;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -141,6 +142,7 @@ public class NodeAttributesManagerImpl extends 
NodeAttributesManager {
 Host node = nodeCollections.get(nodeHost);
 if (node == null) {
   node = new Host(nodeHost);
+  nodeCollections.put(nodeHost, node);
 }
 switch (op) {
 case REMOVE:
@@ -181,8 +183,16 @@ public class NodeAttributesManagerImpl extends 
NodeAttributesManager {
 
   private void removeNodeFromAttributes(String nodeHost,
   Set attributeMappings) {
-for (NodeAttribute attribute : attributeMappings) {
-  clusterAttributes.get(attribute).removeNode(nodeHost);
+for (NodeAttribute rmAttribute : attributeMappings) {
+  RMNodeAttribute host = clusterAttributes.get(rmAttribute);
+  if (host != null) {
+host.removeNode(nodeHost);
+// If there is no other host has such attribute,
+// remove it 

[21/50] [abbrv] hadoop git commit: YARN-8152. Add chart in SLS to illustrate the throughput of the scheduler. Contributed by Tao Yang.

[naganarasimha_gr]

YARN-8152. Add chart in SLS to illustrate the throughput of the scheduler. 
Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba2db39c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba2db39c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba2db39c

Branch: refs/heads/YARN-3409
Commit: ba2db39c655600d00d2d3d3951e0c780f4d6f622
Parents: 71220d2
Author: Weiwei Yang 
Authored: Fri Apr 27 15:36:21 2018 +0800
Committer: Weiwei Yang 
Committed: Fri Apr 27 15:36:21 2018 +0800

--
 .../src/main/html/simulate.html.template| 24 ---
 .../sls/scheduler/SLSCapacityScheduler.java | 29 
 .../yarn/sls/scheduler/SchedulerMetrics.java| 42 
 .../apache/hadoop/yarn/sls/web/SLSWebApp.java   | 70 +++-
 4 files changed, 155 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba2db39c/hadoop-tools/hadoop-sls/src/main/html/simulate.html.template
--
diff --git a/hadoop-tools/hadoop-sls/src/main/html/simulate.html.template 
b/hadoop-tools/hadoop-sls/src/main/html/simulate.html.template
index 1b89713..c8590b0 100644
--- a/hadoop-tools/hadoop-sls/src/main/html/simulate.html.template
+++ b/hadoop-tools/hadoop-sls/src/main/html/simulate.html.template
@@ -36,19 +36,19 @@
 
 
   
-  
+  
 
 
   
-  
+  
 
 
   
-  
+  
 
 
   
-  
+  
 
 
 

[05/50] [abbrv] hadoop git commit: HDFS-13484. RBF: Disable Nameservices from the federation. Contributed by Inigo Goiri.

[naganarasimha_gr]

HDFS-13484. RBF: Disable Nameservices from the federation. Contributed by Inigo 
Goiri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48269c37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48269c37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48269c37

Branch: refs/heads/YARN-3409
Commit: 48269c370c8981244b9d3d5cf1c82a2897ca502e
Parents: ebedd69
Author: Yiqun Lin 
Authored: Wed Apr 25 15:22:26 2018 +0800
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../federation/metrics/FederationMetrics.java   |   2 +-
 .../resolver/ActiveNamenodeResolver.java|   8 +
 .../FederationNamenodeServiceState.java |   3 +-
 .../resolver/MembershipNamenodeResolver.java| 110 ++---
 .../federation/router/RouterAdminServer.java|  50 +++-
 .../router/RouterPermissionChecker.java |  59 -
 .../federation/router/RouterRpcServer.java  |  11 +-
 .../src/site/markdown/HDFSRouterFederation.md   |  11 +
 .../server/federation/FederationTestUtils.java  |  42 
 .../server/federation/MiniRouterDFSCluster.java |   8 +
 .../hdfs/server/federation/MockResolver.java|   6 +
 .../router/TestDisableNameservices.java | 236 +++
 .../federation/router/TestRouterAdmin.java  |  50 +++-
 .../federation/router/TestRouterAdminCLI.java   |  11 +
 .../src/site/markdown/HDFSCommands.md   |   4 +
 15 files changed, 570 insertions(+), 41 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/48269c37/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
index 970bd4f..23f62b6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -642,7 +642,7 @@ public class FederationMetrics implements FederationMBean {
   namenodeResolver.getNamenodesForNameserviceId(nsId);
   if (nns != null) {
 FederationNamenodeContext nn = nns.get(0);
-if (nn != null && nn instanceof MembershipState) {
+if (nn instanceof MembershipState) {
   resultList.add((MembershipState) nn);
 }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/48269c37/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
index 1773b34..f1a5329 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/ActiveNamenodeResolver.java
@@ -109,6 +109,14 @@ public interface ActiveNamenodeResolver {
   Set getNamespaces() throws IOException;
 
   /**
+   * Get a list of all namespaces that are disabled.
+   *
+   * @return List of name spaces identifier in the federation
+   * @throws IOException If the disabled list is not available.
+   */
+  Set getDisabledNamespaces() throws IOException;
+
+  /**
* Assign a unique identifier for the parent router service.
* Required to report the status to the namenode resolver.
*

http://git-wip-us.apache.org/repos/asf/hadoop/blob/48269c37/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
index c773f82..7907e30 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/FederationNamenodeServiceState.java
+++ 

[45/50] [abbrv] hadoop git commit: YARN-8092. Expose Node Attributes info via RM nodes REST API. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-8092. Expose Node Attributes info via RM nodes REST API. Contributed by 
Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/349e6327
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/349e6327
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/349e6327

Branch: refs/heads/YARN-3409
Commit: 349e6327dd8910248f18bba31308762f6aa5ea2a
Parents: 290c087
Author: Sunil G 
Authored: Sun Apr 1 18:07:07 2018 +0530
Committer: Naganarasimha 
Committed: Sat Apr 28 19:36:02 2018 +0800

--
 .../hadoop/yarn/sls/nodemanager/NodeInfo.java   | 13 
 .../yarn/sls/scheduler/RMNodeWrapper.java   | 12 
 .../resourcemanager/ResourceTrackerService.java |  4 ++
 .../server/resourcemanager/rmnode/RMNode.java   | 13 
 .../resourcemanager/rmnode/RMNodeImpl.java  | 15 +
 .../webapp/dao/NodeAttributeInfo.java   | 65 
 .../webapp/dao/NodeAttributesInfo.java  | 49 +++
 .../resourcemanager/webapp/dao/NodeInfo.java| 15 +
 .../yarn/server/resourcemanager/MockNodes.java  | 11 
 9 files changed, 197 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/349e6327/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 0c99139..29d54a1 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -218,6 +219,18 @@ public class NodeInfo {
   return null;
 }
 
+
+@Override
+public void setNodeAttributes(String prefix,
+Set nodeAttributes) {
+
+}
+
+@Override
+public Map getAllNodeAttributes() {
+  return null;
+}
+
 @Override
 public Resource getPhysicalResource() {
   return null;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/349e6327/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 78645e9..a603036 100644
--- 
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ 
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.net.Node;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeState;
 import org.apache.hadoop.yarn.api.records.Resource;
@@ -205,6 +206,17 @@ public class RMNodeWrapper implements RMNode {
   public Map getAllocationTagsWithCount() {
 return node.getAllocationTagsWithCount();
   }
+  
+  @Override
+  public void setNodeAttributes(String prefix,
+  Set nodeAttributes) {
+node.setNodeAttributes(prefix, nodeAttributes);
+  }
+
+  @Override
+  public Map getAllNodeAttributes() {
+return node.getAllNodeAttributes();
+  }
 
   @Override
   public Resource getPhysicalResource() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/349e6327/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
 

[02/50] [abbrv] hadoop git commit: YARN-8140. Improve log message when launch cmd is ran for stopped yarn service. Contributed by Eric Yang

[naganarasimha_gr]

YARN-8140. Improve log message when launch cmd is ran for stopped yarn service. 
Contributed by Eric Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/598483e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/598483e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/598483e4

Branch: refs/heads/YARN-3409
Commit: 598483e496a0e0253310817d8d1bd3511318bc8b
Parents: 4a7369b
Author: Billie Rinaldi 
Authored: Thu Apr 26 12:52:22 2018 -0700
Committer: Owen O'Malley 
Committed: Thu Apr 26 13:54:40 2018 -0700

--
 .../java/org/apache/hadoop/yarn/service/webapp/ApiServer.java | 7 ++-
 1 file changed, 6 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/598483e4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index 6f32598..9a30fcf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -167,8 +167,13 @@ public class ApiServer {
   String message = "Failed to create service " + service.getName()
   + ": {}";
   LOG.error(message, e);
+  if (e.getCause().getMessage().contains("already exists")) {
+message = "Service name " + service.getName() + " is already taken.";
+  } else {
+message = e.getCause().getMessage();
+  }
   return formatResponse(Status.INTERNAL_SERVER_ERROR,
-  e.getCause().getMessage());
+  message);
 }
   }
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[11/50] [abbrv] hadoop git commit: YARN-7939. Added support to upgrade a component instance. Contributed by Chandni Singh

[naganarasimha_gr]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a7369b0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
new file mode 100644
index 000..600e438
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/component/TestComponent.java
@@ -0,0 +1,265 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.component;
+
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.client.api.NMClient;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ServiceContext;
+import org.apache.hadoop.yarn.service.ServiceScheduler;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
+import org.apache.hadoop.yarn.service.TestServiceManager;
+import org.apache.hadoop.yarn.service.api.records.ComponentState;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
+import 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.stubbing.Answer;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+
+import static 
org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType.STOP;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link Component}.
+ */
+public class TestComponent {
+
+  @Rule
+  public ServiceTestUtils.ServiceFSWatcher rule =
+  new ServiceTestUtils.ServiceFSWatcher();
+
+  @Test
+  public void testComponentUpgrade() throws Exception {
+ServiceContext context = createTestContext(rule, "testComponentUpgrade");
+Component comp = context.scheduler.getAllComponents().entrySet().iterator()
+.next().getValue();
+
+ComponentEvent upgradeEvent = new ComponentEvent(comp.getName(),
+ComponentEventType.UPGRADE);
+comp.handle(upgradeEvent);
+Assert.assertEquals("component not in need upgrade state",
+ComponentState.NEEDS_UPGRADE, comp.getComponentSpec().getState());
+  }
+
+  @Test
+  public void testCheckState() throws Exception {
+String serviceName = "testCheckState";
+ServiceContext context = createTestContext(rule, serviceName);
+Component comp = context.scheduler.getAllComponents().entrySet().iterator()
+.next().getValue();
+
+comp.handle(new ComponentEvent(comp.getName(), ComponentEventType.UPGRADE)
+.setTargetSpec(createSpecWithEnv(serviceName, comp.getName(), "key1",
+"val1")).setUpgradeVersion("v2"));
+
+// one instance finished upgrading
+

hadoop git commit: YARN-8104. Add API to fetch node to attribute mapping. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 efcca85f5 -> 341094ca8


YARN-8104. Add API to fetch node to attribute mapping. Contributed by Bibin A 
Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/341094ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/341094ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/341094ca

Branch: refs/heads/YARN-3409
Commit: 341094ca806439461fee31b6dc5d7a73c96b8144
Parents: efcca85
Author: Naganarasimha 
Authored: Fri Apr 20 07:31:53 2018 +0800
Committer: Naganarasimha 
Committed: Fri Apr 20 07:31:53 2018 +0800

--
 .../hadoop/mapred/ResourceMgrDelegate.java  |   6 +
 .../hadoop/mapred/TestClientRedirect.java   |   8 +
 .../yarn/api/ApplicationClientProtocol.java |  18 ++
 .../GetNodesToAttributesRequest.java|  65 +++
 .../GetNodesToAttributesResponse.java   |  63 +++
 .../main/proto/applicationclient_protocol.proto |   1 +
 ..._server_resourcemanager_service_protos.proto |   5 -
 .../src/main/proto/yarn_protos.proto|   5 +
 .../src/main/proto/yarn_service_protos.proto|  10 +-
 .../hadoop/yarn/client/api/YarnClient.java  |  19 ++
 .../yarn/client/api/impl/YarnClientImpl.java|   9 +
 .../ApplicationClientProtocolPBClientImpl.java  |  18 ++
 .../ApplicationClientProtocolPBServiceImpl.java |  21 +++
 .../pb/GetAttributesToNodesResponsePBImpl.java  |   6 +-
 .../pb/GetNodesToAttributesRequestPBImpl.java   | 132 ++
 .../pb/GetNodesToAttributesResponsePBImpl.java  | 181 +++
 .../yarn/nodelabels/NodeAttributesManager.java  |   9 +
 .../impl/pb/NodeToAttributesPBImpl.java |   4 +-
 .../NodesToAttributesMappingRequestPBImpl.java  |   2 +-
 .../hadoop/yarn/api/TestPBImplRecords.java  |  16 +-
 .../yarn/server/MockResourceManagerFacade.java  |   8 +
 .../server/resourcemanager/ClientRMService.java |  13 ++
 .../nodelabels/NodeAttributesManagerImpl.java   |  24 +++
 .../resourcemanager/TestClientRMService.java|  86 +
 .../DefaultClientRequestInterceptor.java|   8 +
 .../clientrm/FederationClientInterceptor.java   |   8 +
 .../router/clientrm/RouterClientRMService.java  |   9 +
 .../PassThroughClientRequestInterceptor.java|   8 +
 28 files changed, 749 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/341094ca/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 4f96a6b..1a7f308 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -551,4 +551,10 @@ public class ResourceMgrDelegate extends YarnClient {
   Set attributes) throws YarnException, IOException {
 return client.getAttributesToNodes(attributes);
   }
+
+  @Override
+  public Map getNodeToAttributes(
+  Set hostNames) throws YarnException, IOException {
+return client.getNodeToAttributes(hostNames);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/341094ca/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
index 23a1a85..5972f65 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
@@ -104,6 +104,8 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
 import 

[1/2] hadoop git commit: YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 5fc916392 -> efcca85f5


http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
index 5ce4803..f6adb43 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/DefaultClientRequestInterceptor.java
@@ -39,8 +39,12 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@@ -327,6 +331,19 @@ public class DefaultClientRequestInterceptor
 return clientRMProxy.getResourceTypeInfo(request);
   }
 
+  @Override
+  public GetAttributesToNodesResponse getAttributesToNodes(
+  GetAttributesToNodesRequest request) throws YarnException, IOException {
+return clientRMProxy.getAttributesToNodes(request);
+  }
+
+  @Override
+  public GetClusterNodeAttributesResponse getClusterNodeAttributes(
+  GetClusterNodeAttributesRequest request)
+  throws YarnException, IOException {
+return clientRMProxy.getClusterNodeAttributes(request);
+  }
+
   @VisibleForTesting
   public void setRMClient(ApplicationClientProtocol clientRM) {
 this.clientRMProxy = clientRM;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index 07eaf97..781ccfc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -44,8 +44,12 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
 import 

[2/2] hadoop git commit: YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.

[naganarasimha_gr]

YARN-8100. Support API interface to query cluster attributes and attribute to 
nodes. Contributed by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efcca85f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efcca85f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efcca85f

Branch: refs/heads/YARN-3409
Commit: efcca85f534790a5f9df09c0f2fa6a772b3d3add
Parents: 5fc9163
Author: Naganarasimha 
Authored: Tue Apr 10 07:28:53 2018 +0800
Committer: Naganarasimha 
Committed: Tue Apr 10 07:28:53 2018 +0800

--
 .../hadoop/mapred/ResourceMgrDelegate.java  |  13 ++
 .../hadoop/mapred/TestClientRedirect.java   |  17 ++
 .../yarn/api/ApplicationClientProtocol.java |  37 
 .../GetAttributesToNodesRequest.java|  70 +++
 .../GetAttributesToNodesResponse.java   |  62 +++
 .../GetClusterNodeAttributesRequest.java|  47 +
 .../GetClusterNodeAttributesResponse.java   |  72 
 .../hadoop/yarn/conf/YarnConfiguration.java |   2 +-
 .../main/proto/applicationclient_protocol.proto |   2 +
 .../src/main/proto/yarn_protos.proto|   4 +
 .../src/main/proto/yarn_service_protos.proto|  15 ++
 .../hadoop/yarn/client/api/YarnClient.java  |  36 +++-
 .../yarn/client/api/impl/YarnClientImpl.java|  21 ++-
 .../ApplicationClientProtocolPBClientImpl.java  |  37 
 .../ApplicationClientProtocolPBServiceImpl.java |  44 +
 .../pb/GetAttributesToNodesRequestPBImpl.java   | 175 ++
 .../pb/GetAttributesToNodesResponsePBImpl.java  | 184 +++
 .../GetClusterNodeAttributesRequestPBImpl.java  |  75 
 .../GetClusterNodeAttributesResponsePBImpl.java | 156 
 .../yarn/nodelabels/NodeAttributesManager.java  |   9 +-
 .../hadoop/yarn/api/TestPBImplRecords.java  |  28 +++
 .../yarn/server/MockResourceManagerFacade.java  |  17 ++
 .../server/resourcemanager/ClientRMService.java |  30 +++
 .../nodelabels/NodeAttributesManagerImpl.java   |  33 ++--
 .../resourcemanager/TestClientRMService.java| 126 +
 .../DefaultClientRequestInterceptor.java|  17 ++
 .../clientrm/FederationClientInterceptor.java   |  17 ++
 .../router/clientrm/RouterClientRMService.java  |  19 ++
 .../PassThroughClientRequestInterceptor.java|  17 ++
 29 files changed, 1361 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index ac4b73b..4f96a6b 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.NodeLabel;
 import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -538,4 +539,16 @@ public class ResourceMgrDelegate extends YarnClient {
   throws YarnException, IOException {
 return client.getResourceTypeInfo();
   }
+
+  @Override
+  public Set getClusterAttributes()
+  throws YarnException, IOException {
+return client.getClusterAttributes();
+  }
+
+  @Override
+  public Map getAttributesToNodes(
+  Set attributes) throws YarnException, IOException {
+return client.getAttributesToNodes(attributes);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
 

[08/50] [abbrv] hadoop git commit: YARN-8018. Added support for initiating yarn service upgrade. Contributed by Chandni Singh

[naganarasimha_gr]

YARN-8018.  Added support for initiating yarn service upgrade.
Contributed by Chandni Singh


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27d60a16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27d60a16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27d60a16

Branch: refs/heads/YARN-3409
Commit: 27d60a16342fd39973d43b61008f54a8815a6237
Parents: edb202e
Author: Eric Yang 
Authored: Mon Mar 26 18:46:31 2018 -0400
Committer: Eric Yang 
Committed: Mon Mar 26 18:46:31 2018 -0400

--
 .../yarn/service/client/ApiServiceClient.java   |  18 ++
 .../hadoop/yarn/service/webapp/ApiServer.java   |  24 ++
 .../hadoop/yarn/service/ClientAMProtocol.java   |  12 +
 .../hadoop/yarn/service/ClientAMService.java|  24 ++
 .../hadoop/yarn/service/ServiceEvent.java   |  49 
 .../hadoop/yarn/service/ServiceEventType.java   |  28 +++
 .../hadoop/yarn/service/ServiceManager.java | 225 +++
 .../hadoop/yarn/service/ServiceScheduler.java   |  19 ++
 .../yarn/service/UpgradeComponentsFinder.java   | 162 +
 .../service/api/records/ComponentState.java |   2 +-
 .../yarn/service/api/records/ServiceState.java  |   2 +-
 .../yarn/service/client/ServiceClient.java  | 153 ++---
 .../yarn/service/component/Component.java   |  14 ++
 .../yarn/service/component/ComponentEvent.java  |  12 +
 .../service/component/ComponentEventType.java   |   4 +-
 .../yarn/service/component/ComponentState.java  |   3 +-
 .../yarn/service/conf/YarnServiceConstants.java |   2 +
 .../pb/client/ClientAMProtocolPBClientImpl.java |  26 +++
 .../service/ClientAMProtocolPBServiceImpl.java  |  24 ++
 .../yarn/service/utils/CoreFileSystem.java  |  31 ++-
 .../yarn/service/utils/ServiceApiUtil.java  |  28 ++-
 .../src/main/proto/ClientAMProtocol.proto   |  19 +-
 .../hadoop/yarn/service/ServiceTestUtils.java   |  90 +++-
 .../TestDefaultUpgradeComponentsFinder.java |  63 ++
 .../hadoop/yarn/service/TestServiceManager.java | 156 +
 .../yarn/service/TestYarnNativeServices.java|  76 ---
 .../yarn/service/client/TestServiceClient.java  | 125 +++
 .../yarn/service/utils/TestCoreFileSystem.java  |  46 
 .../hadoop/yarn/client/api/AppAdminClient.java  |  16 ++
 29 files changed, 1374 insertions(+), 79 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 49702e3..e4a245d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -469,4 +469,22 @@ public class ApiServiceClient extends AppAdminClient {
 return output;
   }
 
+  @Override
+  public int actionUpgrade(String appName,
+  String fileName) throws IOException, YarnException {
+int result;
+try {
+  Service service =
+  loadAppJsonFromLocalFS(fileName, appName, null, null);
+  service.setState(ServiceState.UPGRADING);
+  String buffer = jsonSerDeser.toJson(service);
+  ClientResponse response = getApiClient()
+  .post(ClientResponse.class, buffer);
+  result = processResponse(response);
+} catch (Exception e) {
+  LOG.error("Failed to upgrade application: ", e);
+  result = EXIT_EXCEPTION_THROWN;
+}
+return result;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e7979b8..59ee05d 100644
--- 

[15/50] [abbrv] hadoop git commit: HDFS-13347. RBF: Cache datanode reports. Contributed by Inigo Goiri.

[naganarasimha_gr]

HDFS-13347. RBF: Cache datanode reports. Contributed by Inigo Goiri.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a71656c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a71656c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a71656c1

Branch: refs/heads/YARN-3409
Commit: a71656c1c1bf6c680f1382a76ddcac870061f320
Parents: d1e378d
Author: Yiqun Lin 
Authored: Wed Mar 28 11:00:08 2018 +0800
Committer: Yiqun Lin 
Committed: Wed Mar 28 11:00:08 2018 +0800

--
 .../federation/metrics/FederationMetrics.java   |  2 +-
 .../federation/metrics/NamenodeBeanMetrics.java | 68 +++-
 .../hdfs/server/federation/router/Router.java   | 13 
 .../federation/router/RouterMetricsService.java |  9 +++
 .../federation/router/RouterRpcServer.java  | 16 ++---
 .../server/federation/router/TestRouterRpc.java | 41 +++-
 6 files changed, 135 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
index a80c3be..a99a26a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -434,7 +434,7 @@ public class FederationMetrics implements FederationMBean {
 try {
   RouterRpcServer rpcServer = this.router.getRpcServer();
   DatanodeInfo[] live = rpcServer.getDatanodeReport(
-  DatanodeReportType.LIVE, TIME_OUT);
+  DatanodeReportType.LIVE, false, TIME_OUT);
 
   if (live.length > 0) {
 float totalDfsUsed = 0;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index c4e5b5b..e8c6c82 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -25,6 +25,8 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -32,6 +34,7 @@ import javax.management.NotCompliantMBeanException;
 import javax.management.ObjectName;
 import javax.management.StandardMBean;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
 import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -39,6 +42,7 @@ import 
org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
 import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
 import 
org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
 import org.apache.hadoop.hdfs.server.federation.router.Router;
 import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
 import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
@@ -56,6 +60,10 @@ import org.eclipse.jetty.util.ajax.JSON;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
 /**
  * Expose the Namenode metrics as the Router was one.
  */
@@ -65,6 +73,22 @@ public class NamenodeBeanMetrics
   private static final Logger LOG =
   LoggerFactory.getLogger(NamenodeBeanMetrics.class);
 
+  /** Prevent holding the page from loading too long. */
+  private static final String DN_REPORT_TIME_OUT =
+  

[36/50] [abbrv] hadoop git commit: HADOOP-15253. Should update maxQueueSize when refresh call queue. Contributed by Tao Jie.

[naganarasimha_gr]

HADOOP-15253. Should update maxQueueSize when refresh call queue. Contributed 
by Tao Jie.

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acfd764f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acfd764f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acfd764f

Branch: refs/heads/YARN-3409
Commit: acfd764fcc9990e507c0e7cea746652375aaa632
Parents: 6590747
Author: Konstantin V Shvachko 
Authored: Fri Mar 30 18:55:35 2018 -0700
Committer: Konstantin V Shvachko 
Committed: Fri Mar 30 18:55:35 2018 -0700

--
 .../main/java/org/apache/hadoop/ipc/Server.java|  3 +++
 .../org/apache/hadoop/TestRefreshCallQueue.java| 17 +++--
 2 files changed, 18 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/acfd764f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index d0694fb..c5da3b1 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -657,6 +657,9 @@ public abstract class Server {
   public synchronized void refreshCallQueue(Configuration conf) {
 // Create the next queue
 String prefix = getQueueClassPrefix();
+this.maxQueueSize = handlerCount * conf.getInt(
+CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,
+CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT);
 callQueue.swapQueue(getSchedulerClass(prefix, conf),
 getQueueClass(prefix, conf), maxQueueSize, prefix, conf);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/acfd764f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
index d5eb9cf..faac1c0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
@@ -30,8 +30,10 @@ import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer;
 import org.apache.hadoop.hdfs.tools.DFSAdmin;
@@ -141,8 +143,16 @@ public class TestRefreshCallQueue {
 
 // throw an error when we double-initialize JvmMetrics
 DefaultMetricsSystem.setMiniClusterMode(false);
-
+int serviceHandlerCount = config.getInt(
+DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
+DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
 NameNodeRpcServer rpcServer = (NameNodeRpcServer) cluster.getNameNodeRpc();
+// check callqueue size
+assertEquals(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT
+* serviceHandlerCount, 
rpcServer.getClientRpcServer().getMaxQueueSize());
+// Replace queue and update queue size
+config.setInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,
+150);
 try {
   rpcServer.getClientRpcServer().refreshCallQueue(config);
 } catch (Exception e) {
@@ -158,6 +168,9 @@ public class TestRefreshCallQueue {
 } finally {
   DefaultMetricsSystem.setMiniClusterMode(oldValue);
 }
-  }
+// check callQueueSize has changed
+assertEquals(150 * serviceHandlerCount, rpcServer.getClientRpcServer()
+.getMaxQueueSize());
+ }
 
 }
\ No newline at end of file


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[34/50] [abbrv] hadoop git commit: YARN-7497. Add file system based scheduler configuration store. Contributed by Jiandan Yang.

[naganarasimha_gr]

YARN-7497. Add file system based scheduler configuration store. Contributed by 
Jiandan Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1606dad1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1606dad1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1606dad1

Branch: refs/heads/YARN-3409
Commit: 1606dad133de0dbee59175509a22994ec570ea41
Parents: 6542d17
Author: Weiwei Yang 
Authored: Fri Mar 30 21:41:33 2018 +0800
Committer: Weiwei Yang 
Committed: Fri Mar 30 21:41:33 2018 +0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  16 +-
 .../src/main/resources/yarn-default.xml |  22 ++
 .../scheduler/capacity/CapacityScheduler.java   |   1 +
 .../conf/FSSchedulerConfigurationStore.java | 303 +++
 .../conf/MutableCSConfigurationProvider.java|   3 +
 .../capacity/conf/YarnConfigurationStore.java   |   2 +-
 .../conf/TestFSSchedulerConfigurationStore.java | 173 +++
 .../TestMutableCSConfigurationProvider.java |  50 +++
 8 files changed, 568 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1f62bbd..42f2cae 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -776,12 +776,15 @@ public class YarnConfiguration extends Configuration {
   public static final String MEMORY_CONFIGURATION_STORE = "memory";
   @Private
   @Unstable
-  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+  public static final String FS_CONFIGURATION_STORE = "fs";
   @Private
   @Unstable
   public static final String ZK_CONFIGURATION_STORE = "zk";
   @Private
   @Unstable
+  public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+  @Private
+  @Unstable
   public static final String DEFAULT_CONFIGURATION_STORE =
   FILE_CONFIGURATION_STORE;
   @Private
@@ -809,6 +812,17 @@ public class YarnConfiguration extends Configuration {
   @Private
   @Unstable
   public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
+  @Private
+  @Unstable
+  public static final String SCHEDULER_CONFIGURATION_FS_PATH =
+  YARN_PREFIX + "scheduler.configuration.fs.path";
+  @Private
+  @Unstable
+  public static final String SCHEDULER_CONFIGURATION_FS_MAX_VERSION =
+  YARN_PREFIX + "scheduler.configuration.max.version";
+  @Private
+  @Unstable
+  public static final int DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION = 100;
 
   /** Parent znode path under which ZKConfigurationStore will create znodes. */
   @Private

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 114ba4b..81b6658 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3570,6 +3570,28 @@
 
   
 
+  The file system directory to store the configuration files. The path
+  can be any format as long as it follows hadoop compatible schema,
+  for example value "file:///path/to/dir" means to store files on local
+  file system, value "hdfs:///path/to/dir" means to store files on HDFS.
+  If resource manager HA is enabled, recommended to use hdfs schema so
+  it works in fail-over scenario.
+
+yarn.scheduler.configuration.fs.path
+file://${hadoop.tmp.dir}/yarn/system/schedconf
+  
+
+  
+
+  The max number of configuration file in filesystem.
+  Default is 100 for either.
+
+yarn.scheduler.configuration.max.version
+100
+  
+
+  
+
   ZK root node path for configuration store when using zookeeper-based
   configuration store.
 

[10/50] [abbrv] hadoop git commit: HDFS-13352. RBF: Add xsl stylesheet for hdfs-rbf-default.xml. Contributed by Takanobu Asanuma.

[naganarasimha_gr]

HDFS-13352. RBF: Add xsl stylesheet for hdfs-rbf-default.xml. Contributed by 
Takanobu Asanuma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cef0114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cef0114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cef0114

Branch: refs/heads/YARN-3409
Commit: 4cef01147f98451de8acaa90f42a4d4cd4d0af45
Parents: c22d62b
Author: Inigo Goiri 
Authored: Tue Mar 27 10:31:03 2018 -0700
Committer: Inigo Goiri 
Committed: Tue Mar 27 10:31:03 2018 -0700

--
 hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml |  1 +
 .../src/main/xsl/configuration.xsl  | 37 
 2 files changed, 38 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cef0114/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 9683744..b883726 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -167,6 +167,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd;>
 
   
 
+
   
 
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cef0114/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl
new file mode 100644
index 000..d23168f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl
@@ -0,0 +1,37 @@
+
+
+http://www.w3.org/1999/XSL/Transform; version="1.0">
+
+
+
+
+
+
+ name
+ value
+ description
+
+
+
+  
+  
+  
+
+
+
+
+
+
+


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[17/50] [abbrv] hadoop git commit: YARN-8077. The vmemLimit parameter in ContainersMonitorImpl#isProcessTreeOverLimit is confusing. Contributed by Sen Zhao.

[naganarasimha_gr]

YARN-8077. The vmemLimit parameter in 
ContainersMonitorImpl#isProcessTreeOverLimit is confusing. Contributed by Sen 
Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdee0a4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdee0a4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdee0a4f

Branch: refs/heads/YARN-3409
Commit: cdee0a4f840868d8b8acac15e62da2ab337618c7
Parents: 411993f
Author: Miklos Szegedi 
Authored: Wed Mar 28 09:05:25 2018 -0700
Committer: Miklos Szegedi 
Committed: Wed Mar 28 09:05:25 2018 -0700

--
 .../containermanager/monitor/ContainersMonitorImpl.java | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdee0a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 33986a0..35015c2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -367,7 +367,7 @@ public class ContainersMonitorImpl extends AbstractService 
implements
* @param curMemUsageOfAgedProcesses
*  Memory usage of processes older than an iteration in a container
*  tree
-   * @param vmemLimit
+   * @param memLimit
*  The limit specified for the container
* @return true if the memory usage is more than twice the specified limit,
* or if processes in the tree, older than this thread's monitoring
@@ -376,18 +376,18 @@ public class ContainersMonitorImpl extends 
AbstractService implements
   private boolean isProcessTreeOverLimit(String containerId,
   long currentMemUsage,
   long curMemUsageOfAgedProcesses,
-  long vmemLimit) {
+  long memLimit) {
 boolean isOverLimit = false;
 
-if (currentMemUsage > (2 * vmemLimit)) {
+if (currentMemUsage > (2 * memLimit)) {
   LOG.warn("Process tree for container: " + containerId
-  + " running over twice " + "the configured limit. Limit=" + vmemLimit
+  + " running over twice " + "the configured limit. Limit=" + memLimit
   + ", current usage = " + currentMemUsage);
   isOverLimit = true;
-} else if (curMemUsageOfAgedProcesses > vmemLimit) {
+} else if (curMemUsageOfAgedProcesses > memLimit) {
   LOG.warn("Process tree for container: " + containerId
   + " has processes older than 1 "
-  + "iteration running over the configured limit. Limit=" + vmemLimit
+  + "iteration running over the configured limit. Limit=" + memLimit
   + ", current usage = " + curMemUsageOfAgedProcesses);
   isOverLimit = true;
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[02/50] [abbrv] hadoop git commit: YARN-8068. Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ (Sunil G via wangda)

[naganarasimha_gr]

YARN-8068. Application Priority field causes NPE in app timeline publish when 
Hadoop 2.7 based clients to 2.8+ (Sunil G via wangda)

Change-Id: I7910bd1064a1b4dbbe2084080c060822ea6f3b48


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9eef19b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9eef19b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9eef19b2

Branch: refs/heads/YARN-3409
Commit: 9eef19b2ad78b8464da252d0e23c08675898b9d8
Parents: 5d38157
Author: Wangda Tan 
Authored: Mon Mar 26 11:19:15 2018 -0700
Committer: Wangda Tan 
Committed: Mon Mar 26 11:19:15 2018 -0700

--
 .../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 5 +
 1 file changed, 5 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9eef19b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 61936bb..4234e58 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -455,6 +455,11 @@ public class RMAppImpl implements RMApp, Recoverable {
 if (submissionContext.getPriority() != null) {
   this.applicationPriority = Priority
   .newInstance(submissionContext.getPriority().getPriority());
+} else {
+  // If incoming app does not have priority configured in submission
+  // context, system could be assume that its a 0 priority app and could be
+  // considered as normal.
+  this.applicationPriority = Priority.newInstance(0);
 }
 
 int globalMaxAppAttempts = 
conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[25/50] [abbrv] hadoop git commit: YARN-8076. Support to specify application tags in distributed shell. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-8076. Support to specify application tags in distributed shell. 
Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/431076f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/431076f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/431076f6

Branch: refs/heads/YARN-3409
Commit: 431076f63751f855ab6036ff85825a8552257b93
Parents: 3d185d6
Author: Sunil G 
Authored: Thu Mar 29 15:55:39 2018 +0530
Committer: Sunil G 
Committed: Thu Mar 29 15:55:39 2018 +0530

--
 .../yarn/applications/distributedshell/Client.java   | 15 +++
 1 file changed, 15 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/431076f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index d6a753a..61879d0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -230,6 +230,9 @@ public class Client {
   // Docker client configuration
   private String dockerClientConfig = null;
 
+  // Application tags
+  private Set applicationTags = new HashSet<>();
+
   // Command line options
   private Options opts;
 
@@ -384,6 +387,7 @@ public class Client {
 "Placement specification. Please note, if this option is specified,"
 + " The \"num_containers\" option will be ignored. All requested"
 + " containers will be of type GUARANTEED" );
+opts.addOption("application_tags", true, "Application tags.");
   }
 
   /**
@@ -604,6 +608,14 @@ public class Client {
 if (cliParser.hasOption("docker_client_config")) {
   dockerClientConfig = cliParser.getOptionValue("docker_client_config");
 }
+
+if (cliParser.hasOption("application_tags")) {
+  String applicationTagsStr = cliParser.getOptionValue("application_tags");
+  String[] appTags = applicationTagsStr.split(",");
+  for (String appTag : appTags) {
+this.applicationTags.add(appTag.trim());
+  }
+}
 return true;
   }
 
@@ -729,6 +741,9 @@ public class Client {
 }
 
 Set tags = new HashSet();
+if (applicationTags != null) {
+  tags.addAll(applicationTags);
+}
 if (flowName != null) {
   tags.add(TimelineUtils.generateFlowNameTag(flowName));
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[18/50] [abbrv] hadoop git commit: YARN-6629. NPE occurred when container allocation proposal is applied but its resource requests are removed before. (Tao Yang via wangda)

[naganarasimha_gr]

YARN-6629. NPE occurred when container allocation proposal is applied but its 
resource requests are removed before. (Tao Yang via wangda)

Change-Id: I805880f90b3f6798ec96ed8e8e75755f390a9ad5


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47f711ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47f711ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47f711ee

Branch: refs/heads/YARN-3409
Commit: 47f711eebca315804c80012eea5f31275ac25518
Parents: cdee0a4
Author: Wangda Tan 
Authored: Wed Mar 28 08:47:31 2018 -0700
Committer: Wangda Tan 
Committed: Wed Mar 28 11:07:45 2018 -0700

--
 .../scheduler/capacity/CapacityScheduler.java   |  4 +-
 .../scheduler/common/fica/FiCaSchedulerApp.java | 16 --
 .../capacity/TestCapacityScheduler.java | 53 
 3 files changed, 68 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f711ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index daf0354..bf674a8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2825,8 +2825,8 @@ public class CapacityScheduler extends
   // proposal might be outdated if AM failover just finished
   // and proposal queue was not be consumed in time
   if (app != null && attemptId.equals(app.getApplicationAttemptId())) {
-if (app.accept(cluster, request, updatePending)) {
-  app.apply(cluster, request, updatePending);
+if (app.accept(cluster, request, updatePending)
+&& app.apply(cluster, request, updatePending)) {
   LOG.info("Allocation proposal accepted");
   isSuccess = true;
 } else{

http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f711ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index f3da0a3..32b2cad 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -489,7 +489,7 @@ public class FiCaSchedulerApp extends 
SchedulerApplicationAttempt {
 return accepted;
   }
 
-  public void apply(Resource cluster, ResourceCommitRequest request, boolean updatePending) {
 boolean reReservation = false;
 
@@ -502,8 +502,16 @@ public class FiCaSchedulerApp extends 
SchedulerApplicationAttempt {
 allocation = request.getFirstAllocatedOrReservedContainer();
 SchedulerContainer
 schedulerContainer = allocation.getAllocatedOrReservedContainer();
-RMContainer rmContainer = schedulerContainer.getRmContainer();
 
+// Required sanity check - AM can call 'allocate' to update resource
+// request without locking the scheduler, hence we need to check
+if (updatePending &&
+

[05/50] [abbrv] hadoop git commit: YARN-7794. SLSRunner is not loading timeline service jars, causing failure. (Yufei Gu via Haibo Chen)

[naganarasimha_gr]

YARN-7794. SLSRunner is not loading timeline service jars, causing failure. 
(Yufei Gu via Haibo Chen)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcea5a4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcea5a4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcea5a4d

Branch: refs/heads/YARN-3409
Commit: fcea5a4d798a517443afd0b441883ccc4e9136ba
Parents: 82665a7
Author: Haibo Chen 
Authored: Mon Mar 26 14:30:11 2018 -0700
Committer: Haibo Chen 
Committed: Mon Mar 26 14:30:11 2018 -0700

--
 hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 ++
 1 file changed, 2 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcea5a4d/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
--
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh 
b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index e069a4c..5a9de89 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -77,6 +77,8 @@ function parse_args()
 function calculate_classpath
 {
   hadoop_add_to_classpath_tools hadoop-sls
+
+  hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}/timelineservice"'/*'
 }
 
 function run_simulation() {


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[31/50] [abbrv] hadoop git commit: HDFS-13087. Snapshotted encryption zone information should be immutable. Contributed by LiXin Ge.

[naganarasimha_gr]

HDFS-13087. Snapshotted encryption zone information should be immutable. 
Contributed by LiXin Ge.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c6cfad5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c6cfad5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c6cfad5

Branch: refs/heads/YARN-3409
Commit: 2c6cfad5a31ca4d9126ecd2b3c43cca8543aacb4
Parents: e7e2019
Author: Xiao Chen 
Authored: Thu Mar 29 15:36:31 2018 -0700
Committer: Xiao Chen 
Committed: Thu Mar 29 15:46:34 2018 -0700

--
 .../server/namenode/EncryptionZoneManager.java  | 74 +---
 .../server/namenode/FSDirEncryptionZoneOp.java  | 15 ++--
 .../hdfs/server/namenode/FSDirXAttrOp.java  |  8 ++-
 .../hdfs/server/namenode/XAttrStorage.java  |  9 ++-
 .../apache/hadoop/hdfs/TestEncryptionZones.java | 15 +++-
 5 files changed, 97 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c6cfad5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 176ae1d..b1bca98 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -33,6 +33,9 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.CipherSuite;
 import org.apache.hadoop.crypto.CryptoProtocolVersion;
@@ -50,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
 import org.apache.hadoop.security.AccessControlException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -107,6 +111,34 @@ public class EncryptionZoneManager {
 String getKeyName() {
   return keyName;
 }
+
+@Override
+public boolean equals(Object o) {
+  if (this == o) {
+return true;
+  }
+  if (!(o instanceof EncryptionZoneInt)) {
+return false;
+  }
+
+  EncryptionZoneInt b = (EncryptionZoneInt)o;
+  return new EqualsBuilder()
+  .append(inodeId, b.getINodeId())
+  .append(suite, b.getSuite())
+  .append(version, b.getVersion())
+  .append(keyName, b.getKeyName())
+  .isEquals();
+}
+
+@Override
+public int hashCode() {
+  return new HashCodeBuilder().
+  append(inodeId).
+  append(suite).
+  append(version).
+  append(keyName).
+  toHashCode();
+}
   }
 
   private TreeMap encryptionZones = null;
@@ -315,8 +347,8 @@ public class EncryptionZoneManager {
* 
* Called while holding the FSDirectory lock.
*/
-  boolean isInAnEZ(INodesInPath iip)
-  throws UnresolvedLinkException, SnapshotAccessControlException {
+  boolean isInAnEZ(INodesInPath iip) throws UnresolvedLinkException,
+  SnapshotAccessControlException, IOException {
 assert dir.hasReadLock();
 return (getEncryptionZoneForPath(iip) != null);
   }
@@ -341,7 +373,7 @@ public class EncryptionZoneManager {
* 
* Called while holding the FSDirectory lock.
*/
-  String getKeyName(final INodesInPath iip) {
+  String getKeyName(final INodesInPath iip) throws IOException {
 assert dir.hasReadLock();
 EncryptionZoneInt ezi = getEncryptionZoneForPath(iip);
 if (ezi == null) {
@@ -356,19 +388,43 @@ public class EncryptionZoneManager {
* 
* Called while holding the FSDirectory lock.
*/
-  private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) {
+  private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip)
+  throws  IOException{
 assert dir.hasReadLock();
 Preconditions.checkNotNull(iip);
 if (!hasCreatedEncryptionZone()) {
   return null;
 }
+
+int 

[47/50] [abbrv] hadoop git commit: YARN-7871. Node attributes reporting from NM to RM. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7871. Node attributes reporting from NM to RM. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5bd044e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5bd044e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5bd044e6

Branch: refs/heads/YARN-3409
Commit: 5bd044e6923367be8f5bbade22850478abc4dc27
Parents: cca30fd
Author: Naganarasimha 
Authored: Mon Mar 12 08:05:53 2018 +0800
Committer: Naganarasimha 
Committed: Mon Apr 2 11:43:16 2018 +0800

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  10 +-
 .../yarn/nodelabels/NodeAttributesManager.java  |  17 +-
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   |  19 ++
 .../src/main/resources/yarn-default.xml |  24 +++
 .../yarn/server/nodemanager/NodeManager.java|  70 +--
 .../server/nodemanager/NodeStatusUpdater.java   |  14 ++
 .../nodemanager/NodeStatusUpdaterImpl.java  |  70 ++-
 .../ConfigurationNodeAttributesProvider.java|  90 +
 .../server/nodemanager/TestNodeManager.java |   2 +-
 .../TestNodeStatusUpdaterForLabels.java |  10 +-
 ...TestConfigurationNodeAttributesProvider.java | 185 +++
 .../resourcemanager/ResourceTrackerService.java |  30 +++
 .../nodelabels/NodeAttributesManagerImpl.java   |  52 --
 .../TestResourceTrackerService.java |  78 
 .../nodelabels/TestNodeAttributesManager.java   |  99 ++
 15 files changed, 718 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 730fb25..9d675fe 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3448,9 +3448,12 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_NODE_LABELS_PROVIDER_CONFIG =
   NM_NODE_LABELS_PREFIX + "provider";
 
+  public static final String NM_NODE_ATTRIBUTES_PROVIDER_CONFIG =
+  NM_NODE_ATTRIBUTES_PREFIX + "provider";
+
   // whitelist names for the yarn.nodemanager.node-labels.provider
-  public static final String CONFIG_NODE_LABELS_PROVIDER = "config";
-  public static final String SCRIPT_NODE_LABELS_PROVIDER = "script";
+  public static final String CONFIG_NODE_DESCRIPTOR_PROVIDER = "config";
+  public static final String SCRIPT_NODE_DESCRIPTOR_PROVIDER = "script";
 
   private static final String NM_NODE_LABELS_PROVIDER_PREFIX =
   NM_NODE_LABELS_PREFIX + "provider.";
@@ -3482,6 +3485,9 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_PROVIDER_CONFIGURED_NODE_PARTITION =
   NM_NODE_LABELS_PROVIDER_PREFIX + "configured-node-partition";
 
+  public static final String NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES =
+  NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "configured-node-attributes";
+
   private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX
   + "node-labels.";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
index effda9b..ffa33cf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -35,15 +35,18 @@ public abstract class NodeAttributesManager extends 
AbstractService {
 
   /**
* To completely replace the mappings for a given node with the new Set of
-   * Attributes. If the mapping contains an attribute whose type does not match
-   * a previously existing Attribute under the same prefix (name space) then
-   * exception is thrown. Key would be name of the node and value would be set
-   * of Attributes to 

[26/50] [abbrv] hadoop git commit: HDFS-13357. Improve AclException message "Invalid ACL: only directories may have a default ACL.". Contributed by Gabor Bota.

[naganarasimha_gr]

HDFS-13357. Improve AclException message "Invalid ACL: only directories may 
have a default ACL.". Contributed by Gabor Bota.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad10cbd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad10cbd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad10cbd9

Branch: refs/heads/YARN-3409
Commit: ad10cbd91bcf80e63d5f98868f9258c0119db2a9
Parents: 431076f
Author: Wei-Chiu Chuang 
Authored: Thu Mar 29 07:08:22 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Thu Mar 29 07:08:22 2018 -0700

--
 .../java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad10cbd9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index 58dfe12..10ecc23 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -281,7 +281,8 @@ public final class AclStorage {
   // Only directories may have a default ACL.
   if (!defaultEntries.isEmpty() && !inode.isDirectory()) {
 throw new AclException(
-  "Invalid ACL: only directories may have a default ACL.");
+  "Invalid ACL: only directories may have a default ACL. "
++ "Path: " + inode.getFullPathName());
   }
 
   // Attach entries to the feature.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[42/50] [abbrv] hadoop git commit: YARN-7840. Update PB for prefix support of node attributes. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-7840. Update PB for prefix support of node attributes. Contributed by 
Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d30e12d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d30e12d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d30e12d6

Branch: refs/heads/YARN-3409
Commit: d30e12d691ed694d9348a008b6db3a30f6630bb9
Parents: 9c77100
Author: bibinchundatt 
Authored: Fri Feb 2 10:31:00 2018 +0530
Committer: Naganarasimha 
Committed: Mon Apr 2 08:17:15 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  | 22 ++-
 .../src/main/proto/yarn_protos.proto|  7 ++--
 .../records/impl/pb/NodeAttributePBImpl.java| 39 +---
 .../hadoop/yarn/api/TestPBImplRecords.java  |  7 ++--
 4 files changed, 61 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 13081f3..01c70b2 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -37,15 +37,27 @@ import org.apache.hadoop.yarn.util.Records;
  * Its not compulsory for all the attributes to have value, empty string is the
  * default value of the NodeAttributeType.STRING
  * 
- *
+ * 
+ * Node Attribute Prefix is used as namespace to segregate the attributes.
+ * 
  */
 @Public
 @Unstable
 public abstract class NodeAttribute {
 
+  public static final String DEFAULT_PREFIX = "";
+
   public static NodeAttribute newInstance(String attributeName,
   NodeAttributeType attributeType, String attributeValue) {
+return newInstance(DEFAULT_PREFIX, attributeName, attributeType,
+attributeValue);
+  }
+
+  public static NodeAttribute newInstance(String attributePrefix,
+  String attributeName, NodeAttributeType attributeType,
+  String attributeValue) {
 NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class);
+nodeAttribute.setAttributePrefix(attributePrefix);
 nodeAttribute.setAttributeName(attributeName);
 nodeAttribute.setAttributeType(attributeType);
 nodeAttribute.setAttributeValue(attributeValue);
@@ -54,6 +66,14 @@ public abstract class NodeAttribute {
 
   @Public
   @Unstable
+  public abstract String getAttributePrefix();
+
+  @Public
+  @Unstable
+  public abstract void setAttributePrefix(String attributePrefix);
+
+  @Public
+  @Unstable
   public abstract String getAttributeName();
 
   @Public

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 46acffa..4bd90b0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -376,9 +376,10 @@ enum NodeAttributeTypeProto {
 }
 
 message NodeAttributeProto {
-  optional string attributeName = 1;
-  optional NodeAttributeTypeProto attributeType = 2;
-  optional string attributeValue = 3;
+  optional string attributePrefix = 1;
+  required string attributeName = 2;
+  optional NodeAttributeTypeProto attributeType = 3 [default = STRING];
+  optional string attributeValue = 4 [default=""];
 }
 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 11c9c48..7810939 100644
--- 

[09/50] [abbrv] hadoop git commit: HADOOP-15313. TestKMS should close providers.

[naganarasimha_gr]

HADOOP-15313. TestKMS should close providers.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c22d62b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c22d62b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c22d62b3

Branch: refs/heads/YARN-3409
Commit: c22d62b338cb16d93c4576a9c634041e3610a116
Parents: 27d60a1
Author: Xiao Chen 
Authored: Mon Mar 26 15:59:17 2018 -0700
Committer: Xiao Chen 
Committed: Mon Mar 26 15:59:32 2018 -0700

--
 .../apache/hadoop/io/MultipleIOException.java   | 10 ++
 .../hadoop/crypto/key/kms/server/TestKMS.java   | 38 
 2 files changed, 41 insertions(+), 7 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22d62b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
index 66c1ab1..c9d7ade 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
@@ -76,5 +76,15 @@ public class MultipleIOException extends IOException {
 public IOException build() {
   return createIOException(exceptions);
 }
+
+/**
+ * @return whether any exception was added.
+ */
+public boolean isEmpty() {
+  if (exceptions == null) {
+return true;
+  }
+  return exceptions.isEmpty();
+}
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22d62b3/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git 
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
 
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 1189fbf..1517b04 100644
--- 
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ 
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -35,6 +35,7 @@ import 
org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
 import org.apache.hadoop.crypto.key.kms.ValueQueue;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MultipleIOException;
 import org.apache.hadoop.minikdc.MiniKdc;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
@@ -84,6 +85,7 @@ import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
@@ -111,6 +113,10 @@ public class TestKMS {
 
   private SSLFactory sslFactory;
 
+  // Keep track of all key providers created during a test case, so they can be
+  // closed at test tearDown.
+  private List providersCreated = new LinkedList<>();
+
   @Rule
   public final Timeout testTimeout = new Timeout(18);
 
@@ -144,13 +150,17 @@ public class TestKMS {
 
   protected KeyProvider createProvider(URI uri, Configuration conf)
   throws IOException {
-return new LoadBalancingKMSClientProvider(
-new KMSClientProvider[] { new KMSClientProvider(uri, conf) }, conf);
+final KeyProvider ret = new LoadBalancingKMSClientProvider(
+new KMSClientProvider[] {new KMSClientProvider(uri, conf)}, conf);
+providersCreated.add(ret);
+return ret;
   }
 
   private KMSClientProvider createKMSClientProvider(URI uri, Configuration 
conf)
   throws IOException {
-return new KMSClientProvider(uri, conf);
+final KMSClientProvider ret = new KMSClientProvider(uri, conf);
+providersCreated.add(ret);
+return ret;
   }
 
   protected  T runServer(String keystore, String password, File confDir,
@@ -311,13 +321,28 @@ public class TestKMS {
   }
 
   @After
-  public void tearDownMiniKdc() throws Exception {
+  public void tearDown() throws Exception {
 if (kdc != null) {
   kdc.stop();
   kdc = null;
 }
 UserGroupInformation.setShouldRenewImmediatelyForTests(false);
 UserGroupInformation.reset();
+if (!providersCreated.isEmpty()) {
+  final MultipleIOException.Builder b = new MultipleIOException.Builder();
+  for (KeyProvider kp : providersCreated) {
+try {
+  kp.close();
+} catch 

[32/50] [abbrv] hadoop git commit: HADOOP-12862. LDAP Group Mapping over SSL can not specify trust store. Contributed by Wei-Chiu Chuang and Konstantin Shvachko.

[naganarasimha_gr]

HADOOP-12862. LDAP Group Mapping over SSL can not specify trust store. 
Contributed by Wei-Chiu Chuang and Konstantin Shvachko.

Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2216bde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2216bde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2216bde3

Branch: refs/heads/YARN-3409
Commit: 2216bde322961c0fe33b5822510880a65d5c45fd
Parents: 2c6cfad
Author: Konstantin V Shvachko 
Authored: Thu Mar 29 17:13:18 2018 -0700
Committer: Konstantin V Shvachko 
Committed: Thu Mar 29 17:16:23 2018 -0700

--
 .../org/apache/hadoop/conf/Configuration.java   |  2 +-
 .../hadoop/security/LdapGroupsMapping.java  | 90 +---
 .../src/main/resources/core-default.xml | 21 +
 .../src/site/markdown/GroupsMapping.md  |  3 +
 4 files changed, 105 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2216bde3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 25fd656a..78a2e9f 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2335,7 +2335,7 @@ public class Configuration implements 
Iterable>,
* @return password or null if not found
* @throws IOException
*/
-  protected char[] getPasswordFromCredentialProviders(String name)
+  public char[] getPasswordFromCredentialProviders(String name)
   throws IOException {
 char[] pass = null;
 try {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2216bde3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index babfa38..6beaa9e 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -109,6 +109,27 @@ public class LdapGroupsMapping
   public static final String LDAP_KEYSTORE_PASSWORD_FILE_KEY = 
LDAP_KEYSTORE_PASSWORD_KEY + ".file";
   public static final String LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT = "";
 
+
+  /**
+   * File path to the location of the SSL truststore to use
+   */
+  public static final String LDAP_TRUSTSTORE_KEY = LDAP_CONFIG_PREFIX +
+  ".ssl.truststore";
+
+  /**
+   * The key of the credential entry containing the password for
+   * the LDAP SSL truststore
+   */
+  public static final String LDAP_TRUSTSTORE_PASSWORD_KEY =
+  LDAP_CONFIG_PREFIX +".ssl.truststore.password";
+
+  /**
+   * The path to a file containing the password for
+   * the LDAP SSL truststore
+   */
+  public static final String LDAP_TRUSTSTORE_PASSWORD_FILE_KEY =
+  LDAP_TRUSTSTORE_PASSWORD_KEY + ".file";
+
   /*
* User to bind to the LDAP server with
*/
@@ -226,6 +247,8 @@ public class LdapGroupsMapping
   private boolean useSsl;
   private String keystore;
   private String keystorePass;
+  private String truststore;
+  private String truststorePass;
   private String bindUser;
   private String bindPassword;
   private String userbaseDN;
@@ -526,8 +549,19 @@ public class LdapGroupsMapping
   // Set up SSL security, if necessary
   if (useSsl) {
 env.put(Context.SECURITY_PROTOCOL, "ssl");
-System.setProperty("javax.net.ssl.keyStore", keystore);
-System.setProperty("javax.net.ssl.keyStorePassword", keystorePass);
+if (!keystore.isEmpty()) {
+  System.setProperty("javax.net.ssl.keyStore", keystore);
+}
+if (!keystorePass.isEmpty()) {
+  System.setProperty("javax.net.ssl.keyStorePassword", keystorePass);
+}
+if (!truststore.isEmpty()) {
+  System.setProperty("javax.net.ssl.trustStore", truststore);
+}
+if (!truststorePass.isEmpty()) {
+  System.setProperty("javax.net.ssl.trustStorePassword",
+  truststorePass);
+}
   }
 
   env.put(Context.SECURITY_PRINCIPAL, bindUser);
@@ -572,15 +606,10 @@ public class LdapGroupsMapping
 if (ldapUrl == null 

[07/50] [abbrv] hadoop git commit: YARN-8018. Added support for initiating yarn service upgrade. Contributed by Chandni Singh

[naganarasimha_gr]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 091e624..df6d642 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.yarn.service.api.records.Component;
 import org.apache.hadoop.yarn.service.api.records.Container;
 import org.apache.hadoop.yarn.service.api.records.ContainerState;
 import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
 import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
 import org.hamcrest.CoreMatchers;
 import org.junit.After;
@@ -86,7 +86,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
   @Test (timeout = 20)
   public void testCreateFlexStopDestroyService() throws Exception {
 setupInternal(NUM_NMS);
-ServiceClient client = createClient();
+ServiceClient client = createClient(getConf());
 Service exampleApp = createExampleApplication();
 client.actionCreate(exampleApp);
 SliderFileSystem fileSystem = new SliderFileSystem(getConf());
@@ -141,7 +141,7 @@ public class TestYarnNativeServices extends 
ServiceTestUtils {
   @Test (timeout = 20)
   public void testComponentStartOrder() throws Exception {
 setupInternal(NUM_NMS);
-ServiceClient client = createClient();
+ServiceClient client = createClient(getConf());
 Service exampleApp = new Service();
 exampleApp.setName("teststartorder");
 exampleApp.setVersion("v1");
@@ -169,7 +169,7 @@ public class TestYarnNativeServices extends 
ServiceTestUtils {
 String userB = "userb";
 
 setupInternal(NUM_NMS);
-ServiceClient client = createClient();
+ServiceClient client = createClient(getConf());
 String origBasePath = getConf().get(YARN_SERVICE_BASE_PATH);
 
 Service userAApp = new Service();
@@ -221,7 +221,7 @@ public class TestYarnNativeServices extends 
ServiceTestUtils {
 System.setProperty("user.name", user);
 
 setupInternal(NUM_NMS);
-ServiceClient client = createClient();
+ServiceClient client = createClient(getConf());
 
 Service appA = new Service();
 appA.setName(sameAppName);
@@ -290,7 +290,7 @@ public class TestYarnNativeServices extends 
ServiceTestUtils {
 setConf(conf);
 setupInternal(NUM_NMS);
 
-ServiceClient client = createClient();
+ServiceClient client = createClient(getConf());
 Service exampleApp = createExampleApplication();
 client.actionCreate(exampleApp);
 Multimap containersBeforeFailure =
@@ -333,6 +333,28 @@ public class TestYarnNativeServices extends 
ServiceTestUtils {
 client.actionDestroy(exampleApp.getName());
   }
 
+  @Test(timeout = 20)
+  public void testUpgradeService() throws Exception {
+setupInternal(NUM_NMS);
+ServiceClient client = createClient(getConf());
+
+Service service = createExampleApplication();
+client.actionCreate(service);
+waitForServiceToBeStarted(client, service);
+
+//upgrade the service
+service.setVersion("v2");
+client.actionUpgrade(service);
+
+//wait for service to be in upgrade state
+waitForServiceToBeInState(client, service, ServiceState.UPGRADING);
+SliderFileSystem fs = new SliderFileSystem(getConf());
+Service fromFs = ServiceApiUtil.loadServiceUpgrade(fs,
+service.getName(), service.getVersion());
+Assert.assertEquals(service.getName(), fromFs.getName());
+Assert.assertEquals(service.getVersion(), fromFs.getVersion());
+  }
+
   // Check containers launched are in dependency order
   // Get all containers into a list and sort based on container launch time 
e.g.
   // compa-c1, compa-c2, compb-c1, compb-c2;
@@ -470,16 +492,7 @@ public class TestYarnNativeServices extends 
ServiceTestUtils {
*/
   private void waitForServiceToBeStable(ServiceClient client,
   Service exampleApp) throws TimeoutException, InterruptedException {
-GenericTestUtils.waitFor(() -> {
-  try {
-Service retrievedApp = 

[46/50] [abbrv] hadoop git commit: YARN-8094. Support configuration based Node Attribute provider. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-8094. Support configuration based Node Attribute provider. Contributed by 
Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33b67d03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33b67d03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33b67d03

Branch: refs/heads/YARN-3409
Commit: 33b67d03272e69c9e901a1eae75b2fb0a4a4eed1
Parents: 3a24790
Author: Sunil G 
Authored: Sat Mar 31 19:53:06 2018 +0530
Committer: Naganarasimha 
Committed: Mon Apr 2 11:43:16 2018 +0800

--
 .../ConfigurationNodeAttributesProvider.java| 70 +-
 .../ScriptBasedNodeAttributesProvider.java  |  8 +++
 ...TestConfigurationNodeAttributesProvider.java | 74 
 3 files changed, 150 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/33b67d03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
index 74341eb..ab8a8b1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
@@ -18,13 +18,19 @@
 package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.EnumUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.TimerTask;
 import java.util.Set;
@@ -38,6 +44,9 @@ public class ConfigurationNodeAttributesProvider
   private static final Logger LOG =
   LoggerFactory.getLogger(ConfigurationNodeAttributesProvider.class);
 
+  private static final String NODE_ATTRIBUTES_DELIMITER = ":";
+  private static final String NODE_ATTRIBUTE_DELIMITER = ",";
+
   public ConfigurationNodeAttributesProvider() {
 super("Configuration Based Node Attributes Provider");
   }
@@ -59,11 +68,68 @@ public class ConfigurationNodeAttributesProvider
 setDescriptors(parseAttributes(configuredNodeAttributes));
   }
 
-  // TODO parse attributes from configuration
   @VisibleForTesting
   public Set parseAttributes(String config)
   throws IOException {
-return new HashSet<>();
+if (Strings.isNullOrEmpty(config)) {
+  return ImmutableSet.of();
+}
+Set attributeSet = new HashSet<>();
+// Configuration value should be in one line, format:
+// "ATTRIBUTE_NAME,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE",
+// multiple node-attributes are delimited by ":".
+// Each attribute str should not container any space.
+String[] attributeStrs = config.split(NODE_ATTRIBUTES_DELIMITER);
+for (String attributeStr : attributeStrs) {
+  String[] fields = attributeStr.split(NODE_ATTRIBUTE_DELIMITER);
+  if (fields.length != 3) {
+throw new IOException("Invalid value for "
++ YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
++ "=" + config);
+  }
+
+  // We don't allow user config to overwrite our dist prefix,
+  // so disallow any prefix set in the configuration.
+  if (fields[0].contains("/")) {
+throw new IOException("Node attribute set in "
++ YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
++ " should not contain any prefix.");
+  }
+
+  // Make sure attribute type is valid.
+  if (!EnumUtils.isValidEnum(NodeAttributeType.class, fields[1])) {
+throw new IOException("Invalid node attribute type: "
++ fields[1] + ", valid values are "
++ 

[30/50] [abbrv] hadoop git commit: YARN-8086. ManagedParentQueue with no leaf queues cause JS error in new UI. (Suma Shivaprasad via wangda)

[naganarasimha_gr]

YARN-8086. ManagedParentQueue with no leaf queues cause JS error in new UI. 
(Suma Shivaprasad via wangda)

Change-Id: I6d82af91adec02698afddde6883b1fe1924e6041


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7e20190
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7e20190
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7e20190

Branch: refs/heads/YARN-3409
Commit: e7e20190dface4117abe70dcfd96d19b0e4f5339
Parents: 8fb00c3
Author: Wangda Tan 
Authored: Thu Mar 29 15:03:31 2018 -0700
Committer: Wangda Tan 
Committed: Thu Mar 29 15:03:31 2018 -0700

--
 .../src/main/webapp/app/serializers/yarn-queue/capacity-queue.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7e20190/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index e838255..57714e3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -24,7 +24,7 @@ export default DS.JSONAPISerializer.extend({
 normalizeSingleResponse(store, primaryModelClass, payload, id,
   requestType) {
   var children = [];
-  if (payload.queues) {
+  if (payload.queues && payload.queues.queue) {
 payload.queues.queue.forEach(function(queue) {
   children.push(queue.queueName);
 });
@@ -122,7 +122,7 @@ export default DS.JSONAPISerializer.extend({
   data.push(result.queue);
   includedData = includedData.concat(result.includedData);
 
-  if (payload.queues) {
+  if (payload.queues && payload.queues.queue) {
 for (var i = 0; i < payload.queues.queue.length; i++) {
   var queue = payload.queues.queue[i];
   queue.myParent = payload.queueName;


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[40/50] [abbrv] hadoop git commit: YARN-6855. [YARN-3409] CLI Proto Modifications to support Node Attributes. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-6855. [YARN-3409] CLI Proto Modifications to support Node Attributes. 
Contributed by Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/461dcea8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/461dcea8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/461dcea8

Branch: refs/heads/YARN-3409
Commit: 461dcea80b301307bb57a0e06ebb786a6476c3d1
Parents: dc8e343
Author: Naganarasimha 
Authored: Sun Jan 21 00:53:02 2018 +0800
Committer: Naganarasimha 
Committed: Mon Apr 2 08:17:15 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  |  78 
 .../yarn/api/records/NodeAttributeType.java |  35 
 .../ResourceManagerAdministrationProtocol.java  |  13 +-
 .../AttributeMappingOperationType.java  |  42 
 .../api/protocolrecords/NodeToAttributes.java   |  59 ++
 .../NodesToAttributesMappingRequest.java|  69 +++
 .../NodesToAttributesMappingResponse.java   |  27 +++
 ...esourcemanager_administration_protocol.proto |   1 +
 ..._server_resourcemanager_service_protos.proto |  21 ++
 .../src/main/proto/yarn_protos.proto|  11 ++
 .../records/impl/pb/NodeAttributePBImpl.java| 155 +++
 ...nagerAdministrationProtocolPBClientImpl.java |  26 ++-
 ...agerAdministrationProtocolPBServiceImpl.java |  31 ++-
 .../impl/pb/NodeToAttributesPBImpl.java | 161 +++
 .../NodesToAttributesMappingRequestPBImpl.java  | 194 +++
 .../NodesToAttributesMappingResponsePBImpl.java |  47 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |  34 +++-
 .../yarn/server/MockResourceManagerFacade.java  |  17 +-
 .../server/resourcemanager/AdminService.java|  10 +
 .../DefaultRMAdminRequestInterceptor.java   |   9 +
 .../router/rmadmin/RouterRMAdminService.java|  10 +
 .../PassThroughRMAdminRequestInterceptor.java   |   9 +
 22 files changed, 1044 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
new file mode 100644
index 000..13081f3
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * 
+ * Node Attribute is a kind of a label which represents one of the
+ * attribute/feature of a Node. Its different from node partition label as
+ * resource guarantees across the queues will not be maintained for these type
+ * of labels.
+ * 
+ * 
+ * A given Node can be mapped with any kind of attribute, few examples are
+ * HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS.
+ * 
+ * 
+ * Its not compulsory for all the attributes to have value, empty string is the
+ * default value of the NodeAttributeType.STRING
+ * 
+ *
+ */
+@Public
+@Unstable
+public abstract class NodeAttribute {
+
+  public static NodeAttribute newInstance(String attributeName,
+  NodeAttributeType attributeType, String attributeValue) {
+NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class);
+nodeAttribute.setAttributeName(attributeName);
+nodeAttribute.setAttributeType(attributeType);
+nodeAttribute.setAttributeValue(attributeValue);
+return nodeAttribute;
+  }
+
+  @Public
+  @Unstable
+  public abstract 

[29/50] [abbrv] hadoop git commit: YARN-6257. Fix CapacityScheduler REST API JSON output. Contributed By Tao Yang

[naganarasimha_gr]

YARN-6257. Fix CapacityScheduler REST API JSON output.
   Contributed By Tao Yang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fb00c3f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fb00c3f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fb00c3f

Branch: refs/heads/YARN-3409
Commit: 8fb00c3fce96b445a71fb1a5d98b6a55d6ebfd10
Parents: 9d7a903
Author: Eric Yang 
Authored: Thu Mar 29 17:36:34 2018 -0400
Committer: Eric Yang 
Committed: Thu Mar 29 17:36:34 2018 -0400

--
 .../webapp/dao/CapacitySchedulerHealthInfo.java |  35 +++--
 .../webapp/TestRMWebServicesCapacitySched.java  |   4 +
 .../src/site/markdown/ResourceManagerRest.md| 132 +++
 3 files changed, 158 insertions(+), 13 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
index 652c91b..3683055 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
@@ -25,15 +25,14 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 @XmlAccessorType(XmlAccessType.FIELD)
 public class CapacitySchedulerHealthInfo {
 
   @XmlAccessorType(XmlAccessType.FIELD)
   public static class OperationInformation {
+String operation;
 String nodeId;
 String containerId;
 String queue;
@@ -41,7 +40,9 @@ public class CapacitySchedulerHealthInfo {
 OperationInformation() {
 }
 
-OperationInformation(SchedulerHealth.DetailedInformation di) {
+OperationInformation(String operation,
+SchedulerHealth.DetailedInformation di) {
+  this.operation = operation;
   this.nodeId = di.getNodeId() == null ? "N/A" : di.getNodeId().toString();
   this.containerId =
   di.getContainerId() == null ? "N/A" : di.getContainerId().toString();
@@ -59,6 +60,10 @@ public class CapacitySchedulerHealthInfo {
 public String getQueue() {
   return queue;
 }
+
+public String getOperation() {
+  return operation;
+}
   }
 
   @XmlAccessorType(XmlAccessType.FIELD)
@@ -90,7 +95,7 @@ public class CapacitySchedulerHealthInfo {
   }
 
   long lastrun;
-  Map operationsInfo;
+  List operationsInfo;
   List lastRunDetails;
 
   CapacitySchedulerHealthInfo() {
@@ -100,18 +105,22 @@ public class CapacitySchedulerHealthInfo {
 return lastrun;
   }
 
+  public List getOperationsInfo() {
+return operationsInfo;
+  }
+
   CapacitySchedulerHealthInfo(CapacityScheduler cs) {
 SchedulerHealth ht = cs.getSchedulerHealth();
 lastrun = ht.getLastSchedulerRunTime();
-operationsInfo = new HashMap<>();
-operationsInfo.put("last-allocation",
-  new OperationInformation(ht.getLastAllocationDetails()));
-operationsInfo.put("last-release",
-  new OperationInformation(ht.getLastReleaseDetails()));
-operationsInfo.put("last-preemption",
-  new OperationInformation(ht.getLastPreemptionDetails()));
-operationsInfo.put("last-reservation",
-  new OperationInformation(ht.getLastReservationDetails()));
+operationsInfo = new ArrayList<>();
+operationsInfo.add(new OperationInformation("last-allocation",
+ht.getLastAllocationDetails()));
+operationsInfo.add(
+new OperationInformation("last-release", ht.getLastReleaseDetails()));
+operationsInfo.add(new OperationInformation("last-preemption",
+ht.getLastPreemptionDetails()));
+operationsInfo.add(new OperationInformation("last-reservation",
+ht.getLastReservationDetails()));
 
 lastRunDetails = new ArrayList<>();
 lastRunDetails.add(new 

[41/50] [abbrv] hadoop git commit: YARN-7842. PB changes to carry node-attributes in NM heartbeat. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7842. PB changes to carry node-attributes in NM heartbeat. Contributed by 
Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c771007
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c771007
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c771007

Branch: refs/heads/YARN-3409
Commit: 9c771007360f23817757c5b585f77bbad9092903
Parents: 6308b86
Author: Weiwei Yang 
Authored: Wed Jan 31 20:28:41 2018 +0800
Committer: Naganarasimha 
Committed: Mon Apr 2 08:17:15 2018 +0800

--
 .../protocolrecords/NodeHeartbeatRequest.java   | 17 +++
 .../impl/pb/NodeHeartbeatRequestPBImpl.java | 52 
 .../yarn_server_common_service_protos.proto |  5 ++
 .../protocolrecords/TestProtocolRecords.java| 12 +
 4 files changed, 86 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c771007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
index f238f79..4f99225 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
@@ -28,6 +28,7 @@ import 
org.apache.hadoop.yarn.server.api.records.AppCollectorData;
 import org.apache.hadoop.yarn.server.api.records.MasterKey;
 import org.apache.hadoop.yarn.server.api.records.NodeStatus;
 import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
 
 public abstract class NodeHeartbeatRequest {
   
@@ -61,6 +62,18 @@ public abstract class NodeHeartbeatRequest {
 return nodeHeartbeatRequest;
   }
 
+  public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
+  MasterKey lastKnownContainerTokenMasterKey,
+  MasterKey lastKnownNMTokenMasterKey, Set nodeLabels,
+  Set nodeAttributes,
+  Map registeringCollectors) {
+NodeHeartbeatRequest request = NodeHeartbeatRequest
+.newInstance(nodeStatus, lastKnownContainerTokenMasterKey,
+lastKnownNMTokenMasterKey, nodeLabels, registeringCollectors);
+request.setNodeAttributes(nodeAttributes);
+return request;
+  }
+
   public abstract NodeStatus getNodeStatus();
   public abstract void setNodeStatus(NodeStatus status);
 
@@ -85,4 +98,8 @@ public abstract class NodeHeartbeatRequest {
 
   public abstract void setRegisteringCollectors(Map appCollectorsMap);
+
+  public abstract Set getNodeAttributes();
+
+  public abstract void setNodeAttributes(Set nodeAttributes);
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c771007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index 1ffd223..c59127a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -27,6 +27,9 @@ import java.util.Map;
 import java.util.Set;
 
 import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos;
 import 

[12/50] [abbrv] hadoop git commit: HADOOP-15312. Undocumented KeyProvider configuration keys. Contributed by LiXin Ge.

[naganarasimha_gr]

HADOOP-15312. Undocumented KeyProvider configuration keys. Contributed by LiXin 
Ge.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fe41c65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fe41c65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fe41c65

Branch: refs/heads/YARN-3409
Commit: 3fe41c65d84843f817a4f9ef8999dbf862db6674
Parents: 285bbaa
Author: Wei-Chiu Chuang 
Authored: Tue Mar 27 13:33:41 2018 -0700
Committer: Wei-Chiu Chuang 
Committed: Tue Mar 27 13:33:41 2018 -0700

--
 .../src/main/resources/core-default.xml  | 19 +++
 1 file changed, 19 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fe41c65/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml 
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9074300..49bfa28 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2495,6 +2495,25 @@
 
 
 
+  hadoop.security.key.default.bitlength
+  128
+  
+The length (bits) of keys we want the KeyProvider to produce. Key length
+defines the upper-bound on an algorithm's security, ideally, it would
+coincide with the lower-bound on an algorithm's security.
+  
+
+
+
+  hadoop.security.key.default.cipher
+  AES/CTR/NoPadding
+  
+This indicates the algorithm that be used by KeyProvider for generating
+key, and will be converted to CipherSuite when creating encryption zone.
+  
+
+
+
   fs.har.impl.disable.cache
   true
   Don't cache 'har' filesystem instances.


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[19/50] [abbrv] hadoop git commit: Revert "YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru)."

[naganarasimha_gr]

Revert "YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade 
cache when failing over. (Botong Huang via Subru)."

This reverts commit 2a2ef15caf791f30c471526c1b74e68803f0c405 as 
smart-apply-patch script didn't pick the latest patch.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/725b10e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/725b10e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/725b10e3

Branch: refs/heads/YARN-3409
Commit: 725b10e3aee383d049c97f8ed2b0b1ae873d5ae8
Parents: 47f711e
Author: Subru Krishnan 
Authored: Wed Mar 28 11:26:50 2018 -0700
Committer: Subru Krishnan 
Committed: Wed Mar 28 11:26:50 2018 -0700

--
 .../org/apache/hadoop/yarn/conf/YarnConfiguration.java   |  9 +++--
 .../src/main/resources/yarn-default.xml  |  9 -
 .../failover/FederationRMFailoverProxyProvider.java  | 11 +++
 3 files changed, 6 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/725b10e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 658099a..04b2898 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3089,18 +3089,15 @@ public class YarnConfiguration extends Configuration {
 
   public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
   FEDERATION_PREFIX + "cache-ttl.secs";
-  // 5 minutes
-  public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
-
-  public static final String FEDERATION_FLUSh_CACHE_FOR_RM_ADDR =
-  FEDERATION_PREFIX + "flush-cache-for-rm-addr";
-  public static final boolean DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = 
true;
 
   public static final String FEDERATION_REGISTRY_BASE_KEY =
   FEDERATION_PREFIX + "registry.base-dir";
   public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
   "yarnfederation/";
 
+  // 5 minutes
+  public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
   public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
   FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/725b10e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2eba8df..114ba4b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2925,15 +2925,6 @@
   
 
   
-   
-Whether to flush FederationStateStoreFacade cache to get subcluster info
-when FederationRMFailoverProxyProvider is performing failover.
-   
-   yarn.federation.flush-cache-for-rm-addr
-   true
-  
-
-  
 The registry base directory for federation.
 yarn.federation.registry.base-dir
 yarnfederation/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/725b10e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index b72b199..c631208 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -64,8 

[13/50] [abbrv] hadoop git commit: YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru).

[naganarasimha_gr]

YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache 
when failing over. (Botong Huang via Subru).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a2ef15c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a2ef15c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a2ef15c

Branch: refs/heads/YARN-3409
Commit: 2a2ef15caf791f30c471526c1b74e68803f0c405
Parents: 3fe41c6
Author: Subru Krishnan 
Authored: Tue Mar 27 17:39:46 2018 -0700
Committer: Subru Krishnan 
Committed: Tue Mar 27 17:39:46 2018 -0700

--
 .../org/apache/hadoop/yarn/conf/YarnConfiguration.java   |  9 ++---
 .../src/main/resources/yarn-default.xml  |  9 +
 .../failover/FederationRMFailoverProxyProvider.java  | 11 ---
 3 files changed, 23 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a2ef15c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 04b2898..658099a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3089,15 +3089,18 @@ public class YarnConfiguration extends Configuration {
 
   public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
   FEDERATION_PREFIX + "cache-ttl.secs";
+  // 5 minutes
+  public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
+  public static final String FEDERATION_FLUSh_CACHE_FOR_RM_ADDR =
+  FEDERATION_PREFIX + "flush-cache-for-rm-addr";
+  public static final boolean DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = 
true;
 
   public static final String FEDERATION_REGISTRY_BASE_KEY =
   FEDERATION_PREFIX + "registry.base-dir";
   public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
   "yarnfederation/";
 
-  // 5 minutes
-  public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
-
   public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
   FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a2ef15c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 114ba4b..2eba8df 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2925,6 +2925,15 @@
   
 
   
+   
+Whether to flush FederationStateStoreFacade cache to get subcluster info
+when FederationRMFailoverProxyProvider is performing failover.
+   
+   yarn.federation.flush-cache-for-rm-addr
+   true
+  
+
+  
 The registry base directory for federation.
 yarn.federation.registry.base-dir
 yarnfederation/

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a2ef15c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index c631208..b72b199 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -64,7 +64,8 @@ public class FederationRMFailoverProxyProvider
   private FederationStateStoreFacade facade;
   private SubClusterId 

[16/50] [abbrv] hadoop git commit: YARN-7734. Fix UT failure TestContainerLogsPage#testContainerLogPageAccess. Contributed by Tao Yang.

[naganarasimha_gr]

YARN-7734. Fix UT failure TestContainerLogsPage#testContainerLogPageAccess. 
Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/411993f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/411993f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/411993f6

Branch: refs/heads/YARN-3409
Commit: 411993f6e5723c8cba8100bff0269418e46f6367
Parents: a71656c
Author: Weiwei Yang 
Authored: Wed Mar 28 17:35:17 2018 +0800
Committer: Weiwei Yang 
Committed: Wed Mar 28 17:35:38 2018 +0800

--
 .../yarn/server/nodemanager/webapp/TestContainerLogsPage.java   | 1 +
 1 file changed, 1 insertion(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/411993f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 8dc06c7..ece1af4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
@@ -259,6 +259,7 @@ public class TestContainerLogsPage {
   new ConcurrentHashMap();
   when(context.getContainers()).thenReturn(containers);
   when(context.getLocalDirsHandler()).thenReturn(dirsHandler);
+  when(context.getConf()).thenReturn(conf);
 
   MockContainer container = new MockContainer(appAttemptId,
 new AsyncDispatcher(), conf, user, appId, 1);


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[50/50] [abbrv] hadoop git commit: YARN-8033. CLI Integration with NodeAttributesManagerImpl. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-8033. CLI Integration with NodeAttributesManagerImpl. Contributed by 
Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c20126ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c20126ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c20126ac

Branch: refs/heads/YARN-3409
Commit: c20126acfbc82adce4801f20b764e929d1686c1f
Parents: 4724556
Author: bibinchundatt 
Authored: Sun Apr 1 19:24:00 2018 +0530
Committer: Naganarasimha 
Committed: Mon Apr 2 14:11:12 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  |   3 +-
 .../yarn/client/cli/NodeAttributesCLI.java  |  18 +-
 .../yarn/client/cli/TestNodeAttributesCLI.java  |  18 ++
 .../server/resourcemanager/AdminService.java|  97 +++
 .../resourcemanager/TestRMAdminService.java | 170 +--
 5 files changed, 282 insertions(+), 24 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 4f6846b..25ac9ab 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -45,13 +45,12 @@ import org.apache.hadoop.yarn.util.Records;
 @Unstable
 public abstract class NodeAttribute {
 
-  public static final String DEFAULT_PREFIX = "";
   public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
   public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
 
   public static NodeAttribute newInstance(String attributeName,
   NodeAttributeType attributeType, String attributeValue) {
-return newInstance(DEFAULT_PREFIX, attributeName, attributeType,
+return newInstance(PREFIX_CENTRALIZED, attributeName, attributeType,
 attributeValue);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
index 2eff155..df5a57d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.client.cli;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -50,6 +52,7 @@ import 
org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappin
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 
 /**
  * CLI to map attributes to Nodes.
@@ -311,7 +314,7 @@ public class NodeAttributesCLI extends Configured 
implements Tool {
*/
   private List buildNodeLabelsMapFromStr(String args,
   boolean validateForAttributes, AttributeMappingOperationType operation) {
-List nodeToAttributesList = new ArrayList<>();
+Map nodeToAttributesMap = new HashMap<>();
 for (String nodeToAttributesStr : args.split("[ \n]")) {
   // for each node to attribute mapping
   nodeToAttributesStr = nodeToAttributesStr.trim();
@@ -384,8 +387,9 @@ public class NodeAttributesCLI extends Configured 
implements Tool {
   // TODO when we support different type of attribute type we need to
   // cross verify whether input attributes itself is not violating
   // attribute Name to Type mapping.
-  attributesList.add(NodeAttribute.newInstance(attributeName.trim(),
-  attributeType, attributeValue.trim()));
+  attributesList
+  .add(NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED,
+  attributeName.trim(), 

[35/50] [abbrv] hadoop git commit: HADOOP-15352. Fix default local maven repository path in create-release script. Contributed by Elek, Marton.

[naganarasimha_gr]

HADOOP-15352. Fix default local maven repository path in create-release script. 
Contributed by Elek, Marton.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65907472
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65907472
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65907472

Branch: refs/heads/YARN-3409
Commit: 659074728e3b0985ca38919fb82ec71608a90a69
Parents: 1606dad
Author: Akira Ajisaka 
Authored: Fri Mar 30 23:25:14 2018 +0900
Committer: Akira Ajisaka 
Committed: Fri Mar 30 23:25:14 2018 +0900

--
 dev-support/bin/create-release | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/65907472/dev-support/bin/create-release
--
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index 949a46f..d3d96d9 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -389,7 +389,7 @@ function option_parse
 hadoop_error "ERROR: Cannot set --mvncache and --dockercache 
simultaneously."
 exit 1
   else
-MVNCACHE=${MVNCACHE:-"${HOME}/.m2"}
+MVNCACHE=${MVNCACHE:-"${HOME}/.m2/repository"}
   fi
 
   if [[ "${ASFRELEASE}" = true ]]; then


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[45/50] [abbrv] hadoop git commit: YARN-6858. Attribute Manager to store and provide node attributes in RM. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-6858. Attribute Manager to store and provide node attributes in RM. 
Contributed by Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06fbea4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06fbea4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06fbea4e

Branch: refs/heads/YARN-3409
Commit: 06fbea4e58ffc7d6655effbe486c99b72b577ec0
Parents: 980cc55
Author: Sunil G 
Authored: Fri Feb 23 08:01:58 2018 +0530
Committer: Naganarasimha 
Committed: Mon Apr 2 11:30:55 2018 +0800

--
 .../records/impl/pb/NodeAttributePBImpl.java|  15 +-
 .../hadoop/yarn/nodelabels/AbstractLabel.java   |  71 +++
 .../AttributeExpressionOperation.java   |  26 +
 .../hadoop/yarn/nodelabels/AttributeValue.java  |  53 ++
 .../nodelabels/CommonNodeLabelsManager.java |  24 +-
 .../yarn/nodelabels/NodeAttributesManager.java  |  99 
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   |  97 
 .../hadoop/yarn/nodelabels/RMNodeAttribute.java | 104 
 .../hadoop/yarn/nodelabels/RMNodeLabel.java | 109 ++--
 .../yarn/nodelabels/StringAttributeValue.java   |  61 +++
 .../nodemanager/NodeStatusUpdaterImpl.java  |  16 +-
 .../resourcemanager/RMActiveServiceContext.java |  14 +
 .../yarn/server/resourcemanager/RMContext.java  |   5 +
 .../server/resourcemanager/RMContextImpl.java   |  11 +
 .../server/resourcemanager/ResourceManager.java |  40 +-
 .../nodelabels/NodeAttributesManagerImpl.java   | 527 +++
 .../nodelabels/NodeAttributesStoreEvent.java|  51 ++
 .../NodeAttributesStoreEventType.java   |  26 +
 18 files changed, 1230 insertions(+), 119 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 7810939..12a0ecc 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
@@ -120,7 +120,13 @@ public class NodeAttributePBImpl extends NodeAttribute {
 
   @Override
   public int hashCode() {
-return getProto().hashCode();
+final int prime = 31;
+int result = 1;
+result = prime * result + ((getAttributePrefix() == null) ? 0
+: getAttributePrefix().hashCode());
+result = prime * result
++ ((getAttributeName() == null) ? 0 : getAttributeName().hashCode());
+return result;
   }
 
   @Override
@@ -133,13 +139,10 @@ public class NodeAttributePBImpl extends NodeAttribute {
 }
 if (obj instanceof NodeAttribute) {
   NodeAttribute other = (NodeAttribute) obj;
-  if (!compare(getAttributeName(), other.getAttributeName())) {
-return false;
-  }
-  if (!compare(getAttributeValue(), other.getAttributeValue())) {
+  if (!compare(getAttributePrefix(), other.getAttributePrefix())) {
 return false;
   }
-  if (!compare(getAttributeType(), other.getAttributeType())) {
+  if (!compare(getAttributeName(), other.getAttributeName())) {
 return false;
   }
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
new file mode 100644
index 000..6a44574
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of 

[27/50] [abbrv] hadoop git commit: YARN-8085. ResourceProfilesManager should be set in RMActiveServiceContext. Contributed by Tao Yang.

[naganarasimha_gr]

YARN-8085. ResourceProfilesManager should be set in RMActiveServiceContext. 
Contributed by Tao Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a59d60e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a59d60e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a59d60e

Branch: refs/heads/YARN-3409
Commit: 7a59d60e0c50e8264e07f6b1dadaeedba676c8c2
Parents: ad10cbd
Author: Sunil G 
Authored: Thu Mar 29 21:41:16 2018 +0530
Committer: Sunil G 
Committed: Thu Mar 29 21:41:16 2018 +0530

--
 .../resourcemanager/RMActiveServiceContext.java | 11 ++
 .../server/resourcemanager/RMContextImpl.java   |  6 +--
 .../server/resourcemanager/ResourceManager.java | 13 +++
 .../yarn/server/resourcemanager/TestRMHA.java   | 39 
 4 files changed, 58 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a59d60e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 06a1d00..66065e3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -36,6 +36,7 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
 import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import 
org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor;
@@ -111,6 +112,7 @@ public class RMActiveServiceContext {
   private QueueLimitCalculator queueLimitCalculator;
   private AllocationTagsManager allocationTagsManager;
   private PlacementConstraintManager placementConstraintManager;
+  private ResourceProfilesManager resourceProfilesManager;
 
   public RMActiveServiceContext() {
 queuePlacementManager = new PlacementManager();
@@ -513,4 +515,13 @@ public class RMActiveServiceContext {
   QueueLimitCalculator limitCalculator) {
 this.queueLimitCalculator = limitCalculator;
   }
+
+  public ResourceProfilesManager getResourceProfilesManager() {
+return resourceProfilesManager;
+  }
+
+  public void setResourceProfilesManager(
+  ResourceProfilesManager resourceProfilesManager) {
+this.resourceProfilesManager = resourceProfilesManager;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a59d60e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 0b6be72..84e0f6f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -93,8 +93,6 @@ public class RMContextImpl implements RMContext {
*/
   private RMActiveServiceContext activeServiceContext;
 
-  private ResourceProfilesManager resourceProfilesManager;
-
   private String proxyHostAndPort = null;
 
   /**
@@ -591,7 +589,7 @@ public class RMContextImpl implements RMContext {
 
   @Override
   

[04/50] [abbrv] hadoop git commit: HADOOP-15299. Bump Jackson 2 version to Jackson 2.9.x.

[naganarasimha_gr]

HADOOP-15299. Bump Jackson 2 version to Jackson 2.9.x.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82665a78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82665a78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82665a78

Branch: refs/heads/YARN-3409
Commit: 82665a7887a4bbb3afbc257bec31089173f3a969
Parents: 24bc5e0
Author: Sean Mackrory 
Authored: Thu Mar 8 11:23:23 2018 -0700
Committer: Sean Mackrory 
Committed: Mon Mar 26 14:05:15 2018 -0600

--
 hadoop-project/pom.xml   | 2 +-
 .../java/org/apache/hadoop/tools/rumen/state/StatePool.java  | 8 
 .../java/org/apache/hadoop/tools/rumen/TestHistograms.java   | 3 ++-
 .../yarn/client/api/impl/FileSystemTimelineWriter.java   | 4 +++-
 .../java/org/apache/hadoop/yarn/server/timeline/LogInfo.java | 3 ++-
 .../hadoop/yarn/server/timeline/PluginStoreTestUtils.java| 3 ++-
 .../org/apache/hadoop/yarn/server/timeline/TestLogInfo.java  | 6 --
 7 files changed, 18 insertions(+), 11 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-project/pom.xml
--
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index dcbc484..c093927 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -69,7 +69,7 @@
 
 
 1.9.13
-2.7.8
+2.9.4
 
 
 1.7.25

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
--
diff --git 
a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
 
b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
index 41eb90d..ab6f894 100644
--- 
a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
+++ 
b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
@@ -18,11 +18,11 @@
 package org.apache.hadoop.tools.rumen.state;
 
 import java.io.DataInput;
-import java.io.DataInputStream;
 import java.io.DataOutput;
-import java.io.DataOutputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
@@ -216,7 +216,7 @@ public class StatePool {
 // register the module with the object-mapper
 mapper.registerModule(module);
 
-JsonParser parser = mapper.getFactory().createParser((DataInputStream)in);
+JsonParser parser = mapper.getFactory().createParser((InputStream)in);
 StatePool statePool = mapper.readValue(parser, StatePool.class);
 this.setStates(statePool.getStates());
 parser.close();
@@ -285,7 +285,7 @@ public class StatePool {
 
 JsonFactory outFactory = outMapper.getFactory();
 JsonGenerator jGen =
-outFactory.createGenerator((DataOutputStream)out, JsonEncoding.UTF8);
+outFactory.createGenerator((OutputStream)out, JsonEncoding.UTF8);
 jGen.useDefaultPrettyPrinter();
 
 jGen.writeObject(this);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
--
diff --git 
a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
 
b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
index 52caaf5..1872519 100644
--- 
a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
+++ 
b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.tools.rumen;
 import java.io.IOException;
 
+import java.io.OutputStream;
 import java.util.List;
 
 import com.fasterxml.jackson.core.JsonEncoding;
@@ -142,7 +143,7 @@ public class TestHistograms {
 ObjectMapper mapper = new ObjectMapper();
 JsonFactory factory = mapper.getFactory();
 FSDataOutputStream ostream = lfs.create(goldFilePath, true);
-JsonGenerator gen = factory.createGenerator(ostream,
+JsonGenerator gen = factory.createGenerator((OutputStream)ostream,
 JsonEncoding.UTF8);
 gen.useDefaultPrettyPrinter();
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java

[38/50] [abbrv] hadoop git commit: YARN-6856. [YARN-3409] Support CLI for Node Attributes Mapping. Contributed by Naganarasimha G R.

[naganarasimha_gr]

YARN-6856. [YARN-3409] Support CLI for Node Attributes Mapping. Contributed by 
Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6308b86a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6308b86a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6308b86a

Branch: refs/heads/YARN-3409
Commit: 6308b86a3bcfb1f6605113828332353602eea2f9
Parents: 461dcea
Author: Naganarasimha 
Authored: Tue Jan 23 07:18:20 2018 +0800
Committer: Naganarasimha 
Committed: Mon Apr 2 08:17:15 2018 +0800

--
 .../main/java/org/apache/hadoop/ha/HAAdmin.java |   2 +-
 hadoop-yarn-project/hadoop-yarn/bin/yarn|   5 +
 .../yarn/client/cli/NodeAttributesCLI.java  | 410 +++
 .../yarn/client/cli/TestNodeAttributesCLI.java  | 328 +++
 4 files changed, 744 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index 9b7d7ba..8c92bd0 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -575,7 +575,7 @@ public abstract class HAAdmin extends Configured implements 
Tool {
 return 0;
   }
   
-  protected static class UsageInfo {
+  public static class UsageInfo {
 public final String args;
 public final String help;
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-yarn-project/hadoop-yarn/bin/yarn
--
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn 
b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 69afe6f..7cd838f 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -55,6 +55,7 @@ function hadoop_usage
   hadoop_add_subcommand "timelinereader" client "run the timeline reader 
server"
   hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
   hadoop_add_subcommand "top" client "view cluster information"
+  hadoop_add_subcommand "node-attributes" "map node to attibutes"
   hadoop_add_subcommand "version" client "print the version"
   hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
 }
@@ -186,6 +187,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
   hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
 ;;
+   node-attributes)
+  HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false"
+  HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.NodeAttributesCLI'
+   ;;
 timelineserver)
   HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
   
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
new file mode 100644
index 000..2eff155
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -0,0 +1,410 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package 

[06/50] [abbrv] hadoop git commit: MAPREDUCE-6441. Improve temporary directory name generation in LocalDistributedCacheManager for concurrent processes (wattsinabox, rchiang, haibochen via rkanter)

[naganarasimha_gr]

MAPREDUCE-6441. Improve temporary directory name generation in 
LocalDistributedCacheManager for concurrent processes (wattsinabox, rchiang, 
haibochen via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edb202e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edb202e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edb202e4

Branch: refs/heads/YARN-3409
Commit: edb202e4934be750e43103c047752b97c5eafc94
Parents: fcea5a4
Author: Robert Kanter 
Authored: Mon Mar 26 14:55:53 2018 -0700
Committer: Robert Kanter 
Committed: Mon Mar 26 14:55:53 2018 -0700

--
 .../mapred/LocalDistributedCacheManager.java| 13 ++-
 .../apache/hadoop/mapred/LocalJobRunner.java|  2 +-
 .../TestLocalDistributedCacheManager.java   | 95 +++-
 3 files changed, 78 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/edb202e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
index 2a14ec3..bcf73d1 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
@@ -37,7 +37,7 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.UUID;
 
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.FileUtil;
@@ -82,7 +82,7 @@ class LocalDistributedCacheManager {
* @param conf
* @throws IOException
*/
-  public void setup(JobConf conf) throws IOException {
+  public void setup(JobConf conf, JobID jobId) throws IOException {
 File workDir = new File(System.getProperty("user.dir"));
 
 // Generate YARN local resources objects corresponding to the distributed
@@ -91,9 +91,7 @@ class LocalDistributedCacheManager {
   new LinkedHashMap();
 MRApps.setupDistributedCache(conf, localResources);
 // Generating unique numbers for FSDownload.
-AtomicLong uniqueNumberGenerator =
-new AtomicLong(System.currentTimeMillis());
-
+
 // Find which resources are to be put on the local classpath
 Map classpaths = new HashMap();
 Path[] archiveClassPaths = DistributedCache.getArchiveClassPaths(conf);
@@ -124,9 +122,10 @@ class LocalDistributedCacheManager {
   Path destPath = localDirAllocator.getLocalPathForWrite(".", conf);
   Map resourcesToPaths = Maps.newHashMap();
   for (LocalResource resource : localResources.values()) {
+Path destPathForDownload = new Path(destPath,
+jobId.toString() + "_" + UUID.randomUUID().toString());
 Callable download =
-new FSDownload(localFSFileContext, ugi, conf, new Path(destPath,
-Long.toString(uniqueNumberGenerator.incrementAndGet())),
+new FSDownload(localFSFileContext, ugi, conf, destPathForDownload,
 resource);
 Future future = exec.submit(download);
 resourcesToPaths.put(resource, future);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/edb202e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5e7a250..2ab4e76 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -169,7 +169,7 @@ public class 

[20/50] [abbrv] hadoop git commit: YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru).

[naganarasimha_gr]

YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache 
when failing over. (Botong Huang via Subru).


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d7e

Branch: refs/heads/YARN-3409
Commit: 0d7e014fde717e8b122773b68664f4594106
Parents: 725b10e
Author: Subru Krishnan 
Authored: Wed Mar 28 11:33:19 2018 -0700
Committer: Subru Krishnan 
Committed: Wed Mar 28 11:33:19 2018 -0700

--
 .../hadoop/yarn/conf/YarnConfiguration.java |  9 ++-
 .../yarn/conf/TestYarnConfigurationFields.java  |  2 +
 .../TestFederationRMFailoverProxyProvider.java  | 81 +++-
 .../FederationRMFailoverProxyProvider.java  | 11 ++-
 4 files changed, 94 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 04b2898..1f62bbd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3089,15 +3089,18 @@ public class YarnConfiguration extends Configuration {
 
   public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
   FEDERATION_PREFIX + "cache-ttl.secs";
+  // 5 minutes
+  public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
+  public static final String FEDERATION_FLUSH_CACHE_FOR_RM_ADDR =
+  FEDERATION_PREFIX + "flush-cache-for-rm-addr";
+  public static final boolean DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR = 
true;
 
   public static final String FEDERATION_REGISTRY_BASE_KEY =
   FEDERATION_PREFIX + "registry.base-dir";
   public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
   "yarnfederation/";
 
-  // 5 minutes
-  public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
-
   public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
   FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 9fe4f88..f4d1ac0 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -79,6 +79,8 @@ public class TestYarnConfigurationFields extends 
TestConfigurationFieldsBase {
 .add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+configurationPrefixToSkipCompare
+.add(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR);
 configurationPropsToSkipCompare
 .add(YarnConfiguration.RM_EPOCH);
 configurationPropsToSkipCompare

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
index e3f9155..0a7ee3f 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ 

[01/50] [abbrv] hadoop git commit: YARN-8062. yarn rmadmin -getGroups returns group from which the user has been removed. (Sunil G via wangda) [Forced Update!]

[naganarasimha_gr]

Repository: hadoop
Updated Branches:
  refs/heads/YARN-3409 0df5c70e1 -> c20126acf (forced update)


YARN-8062. yarn rmadmin -getGroups returns group from which the user has been 
removed. (Sunil G via wangda)

Change-Id: I80ed63846502bf7751b890b6c6c6a7c0679e2b4a


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d381570
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d381570
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d381570

Branch: refs/heads/YARN-3409
Commit: 5d381570f83022b411a8740d58486a7f68ab2af6
Parents: 22194f3
Author: Wangda Tan 
Authored: Mon Mar 26 11:16:06 2018 -0700
Committer: Wangda Tan 
Committed: Mon Mar 26 11:16:06 2018 -0700

--
 .../server/resourcemanager/ResourceManager.java | 20 +---
 1 file changed, 17 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d381570/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 5140c9f..38da7f5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -260,9 +260,8 @@ public class ResourceManager extends CompositeService 
implements Recoverable {
 // load core-site.xml
 loadConfigurationXml(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
 
-// Do refreshUserToGroupsMappings with loaded core-site.xml
-Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
-.refresh();
+// Refresh user to group mappings during init.
+refreshUserToGroupMappingsWithConf();
 
 // Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
 // Or use RM specific configurations to overwrite the common ones first
@@ -347,6 +346,21 @@ public class ResourceManager extends CompositeService 
implements Recoverable {
 super.serviceInit(this.conf);
   }
 
+  private void refreshUserToGroupMappingsWithConf()
+  throws YarnException, IOException {
+Configuration newConf = new Configuration(false);
+InputStream confFileInputStream =
+configurationProvider
+.getConfigurationInputStream(newConf, 
YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+if (confFileInputStream != null) {
+  newConf.addResource(confFileInputStream);
+}
+
+// Do refreshUserToGroupsMappings with loaded core-site.xml
+Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(newConf)
+.refresh();
+  }
+
   private void loadConfigurationXml(String configurationFile)
   throws YarnException, IOException {
 InputStream configurationInputStream =


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[24/50] [abbrv] hadoop git commit: YARN-8069. Clean up example hostnames for RegistryDNS. Contributed by Billie Rinaldi

[naganarasimha_gr]

YARN-8069.  Clean up example hostnames for RegistryDNS.
Contributed by Billie Rinaldi


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d185d62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d185d62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d185d62

Branch: refs/heads/YARN-3409
Commit: 3d185d62fc392dd5313f8e36b9674b9200638463
Parents: a991e89
Author: Eric Yang 
Authored: Wed Mar 28 20:17:37 2018 -0400
Committer: Eric Yang 
Committed: Wed Mar 28 20:17:37 2018 -0400

--
 .../registry/server/dns/TestRegistryDNS.java| 66 ++--
 .../test/resources/0.17.172.in-addr.arpa.zone   | 24 +++
 .../markdown/yarn-service/Configurations.md |  4 +-
 3 files changed, 47 insertions(+), 47 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d185d62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index 9c369d0..bce73ad 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -156,7 +156,7 @@ public class TestRegistryDNS extends Assert {
 
   protected Configuration createConfiguration() {
 Configuration conf = new Configuration();
-conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test");
 conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0");
 conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
 return conf;
@@ -179,39 +179,39 @@ public class TestRegistryDNS extends Assert {
 "/registry/users/root/services/org-apache-slider/test1/", record);
 
 // start assessing whether correct records are available
-Record[] recs = assertDNSQuery("test1.root.hwx.test.");
+Record[] recs = assertDNSQuery("test1.root.dev.test.");
 assertEquals("wrong result", "192.168.1.5",
 ((ARecord) recs[0]).getAddress().getHostAddress());
 
-recs = assertDNSQuery("management-api.test1.root.hwx.test.", 2);
-assertEquals("wrong target name", "test1.root.hwx.test.",
+recs = assertDNSQuery("management-api.test1.root.dev.test.", 2);
+assertEquals("wrong target name", "test1.root.dev.test.",
 ((CNAMERecord) recs[0]).getTarget().toString());
 assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
 
-recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.",
+recs = assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.",
 Type.SRV, 1);
 assertTrue("not an SRV record", recs[0] instanceof SRVRecord);
 assertEquals("wrong port", 1026, ((SRVRecord) recs[0]).getPort());
 
-recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", 2);
-assertEquals("wrong target name", "test1.root.hwx.test.",
+recs = assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", 2);
+assertEquals("wrong target name", "test1.root.dev.test.",
 ((CNAMERecord) recs[0]).getTarget().toString());
 assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
 
-recs = assertDNSQuery("http-api.test1.root.hwx.test.", 2);
-assertEquals("wrong target name", "test1.root.hwx.test.",
+recs = assertDNSQuery("http-api.test1.root.dev.test.", 2);
+assertEquals("wrong target name", "test1.root.dev.test.",
 ((CNAMERecord) recs[0]).getTarget().toString());
 assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
 
-recs = assertDNSQuery("http-api.test1.root.hwx.test.", Type.SRV,
+recs = assertDNSQuery("http-api.test1.root.dev.test.", Type.SRV,
 1);
 assertTrue("not an SRV record", recs[0] instanceof SRVRecord);
 assertEquals("wrong port", 1027, ((SRVRecord) recs[0]).getPort());
 
-assertDNSQuery("test1.root.hwx.test.", Type.TXT, 3);
-assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", Type.TXT, 1);
-assertDNSQuery("http-api.test1.root.hwx.test.", Type.TXT, 1);
-assertDNSQuery("management-api.test1.root.hwx.test.", Type.TXT, 1);
+assertDNSQuery("test1.root.dev.test.", Type.TXT, 3);
+assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", Type.TXT, 1);
+assertDNSQuery("http-api.test1.root.dev.test.", 

[23/50] [abbrv] hadoop git commit: HDFS-13314. NameNode should optionally exit if it detects FsImage corruption. Contributed by Arpit Agarwal.

[naganarasimha_gr]

HDFS-13314. NameNode should optionally exit if it detects FsImage corruption. 
Contributed by Arpit Agarwal.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a991e899
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a991e899
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a991e899

Branch: refs/heads/YARN-3409
Commit: a991e899fb9f98d2089f37ac9ac7c485d3bbb959
Parents: 081c350
Author: Arpit Agarwal 
Authored: Wed Mar 28 11:37:34 2018 -0700
Committer: Arpit Agarwal 
Committed: Wed Mar 28 12:49:17 2018 -0700

--
 .../hadoop/hdfs/server/namenode/FSImage.java| 29 +--
 .../server/namenode/FSImageFormatProtobuf.java  | 31 ---
 .../snapshot/FSImageFormatPBSnapshot.java   | 55 ++--
 3 files changed, 101 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a991e899/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index e758108..dd7df5a 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -35,6 +35,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -68,6 +69,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
 import org.apache.hadoop.hdfs.util.Canceler;
 import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.util.Time;
 
 import com.google.common.base.Joiner;
@@ -86,6 +88,10 @@ public class FSImage implements Closeable {
   protected FSEditLog editLog = null;
   private boolean isUpgradeFinalized = false;
 
+  // If true, then image corruption was detected. The NameNode process will
+  // exit immediately after saving the image.
+  private AtomicBoolean exitAfterSave = new AtomicBoolean(false);
+
   protected NNStorage storage;
   
   /**
@@ -954,8 +960,14 @@ public class FSImage implements Closeable {
 
 FSImageFormatProtobuf.Saver saver = new 
FSImageFormatProtobuf.Saver(context);
 FSImageCompression compression = 
FSImageCompression.createCompression(conf);
-saver.save(newFile, compression);
-
+long numErrors = saver.save(newFile, compression);
+if (numErrors > 0) {
+  // The image is likely corrupted.
+  LOG.error("Detected " + numErrors + " errors while saving FsImage " +
+  dstFile);
+  exitAfterSave.set(true);
+}
+
 MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest());
 storage.setMostRecentCheckpointInfo(txid, Time.now());
   }
@@ -1117,6 +1129,12 @@ public class FSImage implements Closeable {
 }
 //Update NameDirSize Metric
 getStorage().updateNameDirSize();
+
+if (exitAfterSave.get()) {
+  LOG.fatal("NameNode process will exit now... The saved FsImage " +
+  nnf + " is potentially corrupted.");
+  ExitUtil.terminate(-1);
+}
   }
 
   /**
@@ -1184,8 +1202,11 @@ public class FSImage implements Closeable {
   
   // Since we now have a new checkpoint, we can clean up some
   // old edit logs and checkpoints.
-  purgeOldStorage(nnf);
-  archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW);
+  // Do not purge anything if we just wrote a corrupted FsImage.
+  if (!exitAfterSave.get()) {
+purgeOldStorage(nnf);
+archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW);
+  }
 } finally {
   // Notify any threads waiting on the checkpoint to be canceled
   // that it is complete.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a991e899/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index cd5a5526..4ac20ad 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ 

[33/50] [abbrv] hadoop git commit: HADOOP-14999. AliyunOSS: provide one asynchronous multi-part based uploading mechanism. Contributed by Genmao Yu.

[naganarasimha_gr]

HADOOP-14999. AliyunOSS: provide one asynchronous multi-part based uploading 
mechanism. Contributed by Genmao Yu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6542d17e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6542d17e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6542d17e

Branch: refs/heads/YARN-3409
Commit: 6542d17ea460ec222137c4b275b13daf15d3fca3
Parents: 2216bde
Author: Sammi Chen 
Authored: Fri Mar 30 20:23:05 2018 +0800
Committer: Sammi Chen 
Committed: Fri Mar 30 20:23:05 2018 +0800

--
 .../aliyun/oss/AliyunCredentialsProvider.java   |   3 +-
 .../aliyun/oss/AliyunOSSBlockOutputStream.java  | 206 +++
 .../fs/aliyun/oss/AliyunOSSFileSystem.java  |  34 ++-
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 173 
 .../fs/aliyun/oss/AliyunOSSOutputStream.java| 111 --
 .../hadoop/fs/aliyun/oss/AliyunOSSUtils.java| 115 ---
 .../apache/hadoop/fs/aliyun/oss/Constants.java  |  22 +-
 .../oss/TestAliyunOSSBlockOutputStream.java | 115 +++
 .../fs/aliyun/oss/TestAliyunOSSInputStream.java |  10 +-
 .../aliyun/oss/TestAliyunOSSOutputStream.java   |  91 
 .../contract/TestAliyunOSSContractDistCp.java   |   2 +-
 11 files changed, 544 insertions(+), 338 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
index b46c67a..58c14a9 100644
--- 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
@@ -35,8 +35,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
 public class AliyunCredentialsProvider implements CredentialsProvider {
   private Credentials credentials = null;
 
-  public AliyunCredentialsProvider(Configuration conf)
-  throws IOException {
+  public AliyunCredentialsProvider(Configuration conf) throws IOException {
 String accessKeyId;
 String accessKeySecret;
 String securityToken;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
--
diff --git 
a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
new file mode 100644
index 000..12d551b
--- /dev/null
+++ 
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.model.PartETag;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Asynchronous multi-part based uploading mechanism to support huge file
+ * which is 

[43/50] [abbrv] hadoop git commit: YARN-7856. Validate Node Attributes from NM. Contributed by Weiwei Yang.

[naganarasimha_gr]

YARN-7856. Validate Node Attributes from NM. Contributed by Weiwei Yang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14a92509
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14a92509
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14a92509

Branch: refs/heads/YARN-3409
Commit: 14a9250912a75bb8964da8c260286e3763edc666
Parents: 06fbea4
Author: Sunil G 
Authored: Tue Feb 27 08:15:42 2018 +0530
Committer: Naganarasimha 
Committed: Mon Apr 2 11:30:55 2018 +0800

--
 .../hadoop/yarn/api/records/NodeAttribute.java  |  2 ++
 .../hadoop/yarn/nodelabels/NodeLabelUtil.java   | 31 
 .../ScriptBasedNodeAttributesProvider.java  | 25 ++--
 .../TestScriptBasedNodeAttributesProvider.java  | 27 +
 4 files changed, 83 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 01c70b2..4f6846b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.yarn.util.Records;
 public abstract class NodeAttribute {
 
   public static final String DEFAULT_PREFIX = "";
+  public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
+  public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
 
   public static NodeAttribute newInstance(String attributeName,
   NodeAttributeType attributeType, String attributeValue) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
index d918712..fdfd0ce 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -17,7 +17,11 @@
  */
 package org.apache.hadoop.yarn.nodelabels;
 
+import com.google.common.base.Strings;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
 import java.io.IOException;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 /**
@@ -94,4 +98,31 @@ public final class NodeLabelUtil {
   + ", now it is= " + prefix);
 }
   }
+
+  /**
+   * Validate if a given set of attributes are valid. Attributes could be
+   * invalid if any of following conditions is met:
+   *
+   * 
+   *   Missing prefix: the attribute doesn't have prefix defined
+   *   Malformed attribute prefix: the prefix is not in valid format
+   * 
+   * @param attributeSet
+   * @throws IOException
+   */
+  public static void validateNodeAttributes(Set attributeSet)
+  throws IOException {
+if (attributeSet != null && !attributeSet.isEmpty()) {
+  for (NodeAttribute nodeAttribute : attributeSet) {
+String prefix = nodeAttribute.getAttributePrefix();
+if (Strings.isNullOrEmpty(prefix)) {
+  throw new IOException("Attribute prefix must be set");
+}
+// Verify attribute prefix format.
+checkAndThrowAttributePrefix(prefix);
+// Verify attribute name format.
+checkAndThrowLabelName(nodeAttribute.getAttributeName());
+  }
+}
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
 

[37/50] [abbrv] hadoop git commit: HDFS-13289. RBF: TestConnectionManager#testCleanup() test case need correction. Contributed by Dibyendu Karmakar.

[naganarasimha_gr]

HDFS-13289. RBF: TestConnectionManager#testCleanup() test case need correction. 
Contributed by Dibyendu Karmakar.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc8e3432
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc8e3432
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc8e3432

Branch: refs/heads/YARN-3409
Commit: dc8e3432013153ac11d31d6b462aa96f8ca2c443
Parents: acfd764
Author: Inigo Goiri 
Authored: Sat Mar 31 09:46:13 2018 -0700
Committer: Inigo Goiri 
Committed: Sat Mar 31 09:46:13 2018 -0700

--
 .../router/TestConnectionManager.java   | 22 +++-
 1 file changed, 17 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc8e3432/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
index 741d1f6..2e4b80d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
@@ -29,6 +29,7 @@ import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * Test functionalities of {@link ConnectionManager}, which manages a pool
@@ -94,14 +95,20 @@ public class TestConnectionManager {
 // Make sure the number of connections doesn't go below minSize
 ConnectionPool pool3 = new ConnectionPool(
 conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10);
-addConnectionsToPool(pool3, 10, 0);
-poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool3);
-connManager.cleanup(pool3);
+addConnectionsToPool(pool3, 8, 0);
+poolMap.put(new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS), pool3);
+checkPoolConnections(TEST_USER3, 10, 0);
+for (int i = 0; i < 10; i++) {
+  connManager.cleanup(pool3);
+}
 checkPoolConnections(TEST_USER3, 2, 0);
 // With active connections added to pool, make sure it honors the
 // MIN_ACTIVE_RATIO again
-addConnectionsToPool(pool3, 10, 2);
-connManager.cleanup(pool3);
+addConnectionsToPool(pool3, 8, 2);
+checkPoolConnections(TEST_USER3, 10, 2);
+for (int i = 0; i < 10; i++) {
+  connManager.cleanup(pool3);
+}
 checkPoolConnections(TEST_USER3, 4, 2);
   }
 
@@ -145,13 +152,18 @@ public class TestConnectionManager {
 
   private void checkPoolConnections(UserGroupInformation ugi,
   int numOfConns, int numOfActiveConns) {
+boolean connPoolFoundForUser = false;
 for (Map.Entry e :
 connManager.getPools().entrySet()) {
   if (e.getKey().getUgi() == ugi) {
 assertEquals(numOfConns, e.getValue().getNumConnections());
 assertEquals(numOfActiveConns, e.getValue().getNumActiveConnections());
+connPoolFoundForUser = true;
   }
 }
+if (!connPoolFoundForUser) {
+  fail("Connection pool not found for user " + ugi.getUserName());
+}
   }
 
 }


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org

[28/50] [abbrv] hadoop git commit: HADOOP-15342. Updating ADLS connector to use the current SDK version (2.2.7). Contributed by Atul Sikaria.

[naganarasimha_gr]

HADOOP-15342. Updating ADLS connector to use the current SDK version (2.2.7). 
Contributed by Atul Sikaria.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d7a9031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d7a9031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d7a9031

Branch: refs/heads/YARN-3409
Commit: 9d7a9031a5978efc8d97566e35ebaace20db2353
Parents: 7a59d60
Author: Sean Mackrory 
Authored: Tue Mar 27 15:51:44 2018 -0600
Committer: Sean Mackrory 
Committed: Thu Mar 29 10:13:40 2018 -0600

--
 hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d7a9031/hadoop-tools/hadoop-azure-datalake/pom.xml
--
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml 
b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 8020381..57515b0 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -33,7 +33,7 @@
 0.9.1
 UTF-8
 true
-
2.2.5
+
2.2.7
   
   
 


-
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org