Repository: hadoop Updated Branches: refs/heads/branch-2 48d8d8824 -> 0ec6e7e7f
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java index 7bf92af..1ff0bef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/AddToClusterNodeLabelsRequestPBImpl.java @@ -18,23 +18,27 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; -import java.util.HashSet; -import java.util.Set; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.apache.hadoop.yarn.api.records.NodeLabel; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProtoOrBuilder; import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest; public class AddToClusterNodeLabelsRequestPBImpl extends AddToClusterNodeLabelsRequest { - Set<String> labels; AddToClusterNodeLabelsRequestProto proto = AddToClusterNodeLabelsRequestProto .getDefaultInstance(); AddToClusterNodeLabelsRequestProto.Builder builder = null; + private List<NodeLabel> updatedNodeLabels; boolean viaProto = false; public AddToClusterNodeLabelsRequestPBImpl() { - this.builder = AddToClusterNodeLabelsRequestProto.newBuilder(); + builder = AddToClusterNodeLabelsRequestProto.newBuilder(); } public AddToClusterNodeLabelsRequestPBImpl( @@ -43,18 +47,11 @@ public class AddToClusterNodeLabelsRequestPBImpl extends viaProto = true; } - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = AddToClusterNodeLabelsRequestProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.labels != null && !this.labels.isEmpty()) { - builder.clearNodeLabels(); - builder.addAllNodeLabels(this.labels); - } + public AddToClusterNodeLabelsRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; } private void mergeLocalToProto() { @@ -65,35 +62,30 @@ public class AddToClusterNodeLabelsRequestPBImpl extends viaProto = true; } - public AddToClusterNodeLabelsRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void initLabels() { - if (this.labels != null) { - return; + private void mergeLocalToBuilder() { + if (this.updatedNodeLabels != null) { + addNodeLabelsToProto(); } - AddToClusterNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; - this.labels = new HashSet<String>(); - this.labels.addAll(p.getNodeLabelsList()); } - @Override - public void setNodeLabels(Set<String> labels) { + private void addNodeLabelsToProto() { maybeInitBuilder(); - if (labels == null || labels.isEmpty()) { - builder.clearNodeLabels(); + builder.clearNodeLabels(); + List<NodeLabelProto> protoList = new ArrayList<NodeLabelProto>(); + for (NodeLabel r : this.updatedNodeLabels) { + protoList.add(convertToProtoFormat(r)); } - this.labels = labels; + builder.addAllNodeLabels(protoList); } @Override - public Set<String> getNodeLabels() { - initLabels(); - return this.labels; + public boolean equals(Object other) { + if (other == null) + return false; + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; } @Override @@ -101,14 +93,53 @@ public class AddToClusterNodeLabelsRequestPBImpl extends assert false : "hashCode not designed"; return 0; } - + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = AddToClusterNodeLabelsRequestProto.newBuilder(proto); + } + viaProto = false; + } + @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); + public void setNodeLabels(List<NodeLabel> updatedNodeLabels) { + maybeInitBuilder(); + this.updatedNodeLabels = new ArrayList<>(); + if (updatedNodeLabels == null) { + builder.clearNodeLabels(); + return; } - return false; + this.updatedNodeLabels.addAll(updatedNodeLabels); + } + + private void initLocalNodeLabels() { + AddToClusterNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; + List<NodeLabelProto> attributesProtoList = p.getNodeLabelsList(); + this.updatedNodeLabels = new ArrayList<NodeLabel>(); + for (NodeLabelProto r : attributesProtoList) { + this.updatedNodeLabels.add(convertFromProtoFormat(r)); + } + } + + @Override + public List<NodeLabel> getNodeLabels() { + if (this.updatedNodeLabels != null) { + return this.updatedNodeLabels; + } + initLocalNodeLabels(); + return this.updatedNodeLabels; + } + + private NodeLabel convertFromProtoFormat(NodeLabelProto p) { + return new NodeLabelPBImpl(p); + } + + private NodeLabelProto convertToProtoFormat(NodeLabel t) { + return ((NodeLabelPBImpl) t).getProto(); + } + + @Override + public String toString() { + return getProto().toString(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java deleted file mode 100644 index f18a8db..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsRequestPBImpl.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.yarn.api.records.NodeLabel; -import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProtoOrBuilder; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; - -public class UpdateNodeLabelsRequestPBImpl extends - UpdateNodeLabelsRequest { - UpdateNodeLabelsRequestProto proto = - UpdateNodeLabelsRequestProto.getDefaultInstance(); - UpdateNodeLabelsRequestProto.Builder builder = null; - private List<NodeLabel> updatedNodeLabels; - boolean viaProto = false; - - public UpdateNodeLabelsRequestPBImpl() { - builder = UpdateNodeLabelsRequestProto.newBuilder(); - } - - public UpdateNodeLabelsRequestPBImpl( - UpdateNodeLabelsRequestProto proto) { - this.proto = proto; - viaProto = true; - } - - public UpdateNodeLabelsRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) - maybeInitBuilder(); - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void mergeLocalToBuilder() { - if (this.updatedNodeLabels != null) { - addNodeLabelsToProto(); - } - } - - private void addNodeLabelsToProto() { - maybeInitBuilder(); - builder.clearNodeLabels(); - List<NodeLabelProto> protoList = - new ArrayList<NodeLabelProto>(); - for (NodeLabel r : this.updatedNodeLabels) { - protoList.add(convertToProtoFormat(r)); - } - builder.addAllNodeLabels(protoList); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public int hashCode() { - assert false : "hashCode not designed"; - return 0; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = UpdateNodeLabelsRequestProto.newBuilder(proto); - } - viaProto = false; - } - - @Override - public void setNodeLabels(List<NodeLabel> updatedNodeLabels) { - maybeInitBuilder(); - if (updatedNodeLabels == null) { - builder.clearNodeLabels(); - } - this.updatedNodeLabels = updatedNodeLabels; - } - - private void initLocalNodeLabels() { - UpdateNodeLabelsRequestProtoOrBuilder p = viaProto ? proto : builder; - List<NodeLabelProto> attributesProtoList = - p.getNodeLabelsList(); - this.updatedNodeLabels = new ArrayList<NodeLabel>(); - for (NodeLabelProto r : attributesProtoList) { - this.updatedNodeLabels.add(convertFromProtoFormat(r)); - } - } - - @Override - public List<NodeLabel> getNodeLabels() { - if (this.updatedNodeLabels != null) { - return this.updatedNodeLabels; - } - initLocalNodeLabels(); - return this.updatedNodeLabels; - } - - private NodeLabel - convertFromProtoFormat(NodeLabelProto p) { - return new NodeLabelPBImpl(p); - } - - private NodeLabelProto convertToProtoFormat(NodeLabel t) { - return ((NodeLabelPBImpl) t).getProto(); - } - - @Override - public String toString() { - return getProto().toString(); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java deleted file mode 100644 index 217b8fc..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeLabelsResponsePBImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb; - -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsResponseProto; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; - -public class UpdateNodeLabelsResponsePBImpl extends - UpdateNodeLabelsResponse { - UpdateNodeLabelsResponseProto proto = - UpdateNodeLabelsResponseProto.getDefaultInstance(); - UpdateNodeLabelsResponseProto.Builder builder = null; - boolean viaProto = false; - - public UpdateNodeLabelsResponsePBImpl() { - builder = UpdateNodeLabelsResponseProto.newBuilder(); - } - - public UpdateNodeLabelsResponsePBImpl( - UpdateNodeLabelsResponseProto proto) { - this.proto = proto; - viaProto = true; - } - - public UpdateNodeLabelsResponseProto getProto() { - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) - return false; - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return getProto().toString().replaceAll("\\n", ", ") - .replaceAll("\\s+", " "); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 931bca4..8dacd3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -233,8 +233,6 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Remov import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeResponseProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsRequestProto; -import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; @@ -311,8 +309,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClust import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsRequestPBImpl; -import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl; import org.apache.hadoop.yarn.util.resource.Resources; @@ -1295,18 +1291,6 @@ public class TestPBImplRecords { } @Test - public void testUpdateNodeLabelsRequestPBImpl() throws Exception { - validatePBImplRecord(UpdateNodeLabelsRequestPBImpl.class, - UpdateNodeLabelsRequestProto.class); - } - - @Test - public void testUpdateNodeLabelsResponsePBImpl() throws Exception { - validatePBImplRecord(UpdateNodeLabelsResponsePBImpl.class, - UpdateNodeLabelsResponseProto.class); - } - - @Test public void testCheckForDecommissioningNodesRequestPBImpl() throws Exception { validatePBImplRecord(CheckForDecommissioningNodesRequestPBImpl.class, CheckForDecommissioningNodesRequestProto.class); @@ -1317,4 +1301,4 @@ public class TestPBImplRecords { validatePBImplRecord(CheckForDecommissioningNodesResponsePBImpl.class, CheckForDecommissioningNodesResponseProto.class); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java index 67e6119..48d6dc8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java @@ -31,9 +31,8 @@ import org.apache.hadoop.yarn.event.InlineDispatcher; public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager { Map<NodeId, Set<String>> lastNodeToLabels = null; - Collection<String> lastAddedlabels = null; + Collection<NodeLabel> lastAddedlabels = null; Collection<String> lastRemovedlabels = null; - List<NodeLabel> lastUpdatedNodeLabels = null; @Override public void initNodeLabelStore(Configuration conf) { @@ -56,17 +55,11 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager { } @Override - public void storeNewClusterNodeLabels(Set<String> label) throws IOException { + public void storeNewClusterNodeLabels(List<NodeLabel> label) throws IOException { lastAddedlabels = label; } @Override - public void updateNodeLabels(List<NodeLabel> updatedNodeLabels) - throws IOException { - lastUpdatedNodeLabels = updatedNodeLabels; - } - - @Override public void close() throws IOException { // do nothing } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java index 3b9825d..4d406fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java @@ -88,6 +88,7 @@ public class NodeLabelTestBase { Assert.assertTrue(s1.containsAll(s2)); } + @SuppressWarnings("unchecked") public static <E> Set<E> toSet(E... elements) { Set<E> set = Sets.newHashSet(elements); return set; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index 1e2326b..beb2cf8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -31,7 +32,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.exceptions.YarnException; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -61,15 +61,14 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testAddRemovelabel() throws Exception { // Add some label - mgr.addToCluserNodeLabels(ImmutableSet.of("hello")); - assertCollectionEquals(mgr.lastAddedlabels, Arrays.asList("hello")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("hello")); + verifyNodeLabelAdded(Sets.newHashSet("hello"), mgr.lastAddedlabels); - mgr.addToCluserNodeLabels(ImmutableSet.of("world")); - mgr.addToCluserNodeLabels(toSet("hello1", "world1")); - assertCollectionEquals(mgr.lastAddedlabels, - Sets.newHashSet("hello1", "world1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("world")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("hello1", "world1")); + verifyNodeLabelAdded(Sets.newHashSet("hello1", "world1"), mgr.lastAddedlabels); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Sets.newHashSet("hello", "world", "hello1", "world1"))); // try to remove null, empty and non-existed label, should fail @@ -86,23 +85,32 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { // Remove some label mgr.removeFromClusterNodeLabels(Arrays.asList("hello")); - assertCollectionEquals(mgr.lastRemovedlabels, Arrays.asList("hello")); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + assertCollectionEquals(Sets.newHashSet("hello"), mgr.lastRemovedlabels); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("world", "hello1", "world1"))); mgr.removeFromClusterNodeLabels(Arrays .asList("hello1", "world1", "world")); Assert.assertTrue(mgr.lastRemovedlabels.containsAll(Sets.newHashSet( "hello1", "world1", "world"))); - Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty()); } @Test(timeout = 5000) public void testAddlabelWithCase() throws Exception { // Add some label, case will not ignore here - mgr.addToCluserNodeLabels(ImmutableSet.of("HeLlO")); - assertCollectionEquals(mgr.lastAddedlabels, Arrays.asList("HeLlO")); - Assert.assertFalse(mgr.getClusterNodeLabels().containsAll(Arrays.asList("hello"))); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("HeLlO")); + verifyNodeLabelAdded(Sets.newHashSet("HeLlO"), mgr.lastAddedlabels); + Assert.assertFalse(mgr.getClusterNodeLabelNames().containsAll( + Arrays.asList("hello"))); + } + + @Test(timeout = 5000) + public void testAddlabelWithExclusivity() throws Exception { + // Add some label, case will not ignore here + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("a", false), NodeLabel.newInstance("b", true))); + Assert.assertFalse(mgr.isExclusiveNodeLabel("a")); + Assert.assertTrue(mgr.isExclusiveNodeLabel("b")); } @Test(timeout = 5000) @@ -111,7 +119,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { try { Set<String> set = new HashSet<String>(); set.add(null); - mgr.addToCluserNodeLabels(set); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(set); } catch (IOException e) { caught = true; } @@ -119,7 +127,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of(CommonNodeLabelsManager.NO_LABEL)); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of(CommonNodeLabelsManager.NO_LABEL)); } catch (IOException e) { caught = true; } @@ -128,7 +136,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("-?")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("-?")); } catch (IOException e) { caught = true; } @@ -136,7 +144,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of(StringUtils.repeat("c", 257))); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of(StringUtils.repeat("c", 257))); } catch (IOException e) { caught = true; } @@ -144,7 +152,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("-aaabbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("-aaabbb")); } catch (IOException e) { caught = true; } @@ -152,7 +160,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("_aaabbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("_aaabbb")); } catch (IOException e) { caught = true; } @@ -160,7 +168,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("a^aabbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("a^aabbb")); } catch (IOException e) { caught = true; } @@ -168,7 +176,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { caught = false; try { - mgr.addToCluserNodeLabels(ImmutableSet.of("aa[a]bbb")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("aa[a]bbb")); } catch (IOException e) { caught = true; } @@ -198,7 +206,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { Assert.assertTrue("trying to add a empty node but succeeded", caught); // set node->label one by one - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2"), toSet("p3"))); @@ -249,7 +257,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testRemovelabelWithNodes() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n3"), toSet("p3"))); @@ -261,21 +269,21 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { mgr.removeFromClusterNodeLabels(ImmutableSet.of("p2", "p3")); Assert.assertTrue(mgr.getNodeLabels().isEmpty()); - Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty()); assertCollectionEquals(mgr.lastRemovedlabels, Arrays.asList("p2", "p3")); } @Test(timeout = 5000) public void testTrimLabelsWhenAddRemoveNodeLabels() throws IOException { - mgr.addToCluserNodeLabels(toSet(" p1")); - assertCollectionEquals(mgr.getClusterNodeLabels(), toSet("p1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet(" p1")); + assertCollectionEquals(mgr.getClusterNodeLabelNames(), toSet("p1")); mgr.removeFromClusterNodeLabels(toSet("p1 ")); - Assert.assertTrue(mgr.getClusterNodeLabels().isEmpty()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().isEmpty()); } @Test(timeout = 5000) public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException { - mgr.addToCluserNodeLabels(toSet(" p1", "p2")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet(" p1", "p2")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 "))); assertMapEquals( mgr.getNodeLabels(), @@ -291,7 +299,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo() throws IOException { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); assertMapEquals( mgr.getNodeLabels(), @@ -330,7 +338,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { // add labels try { - mgr.addToCluserNodeLabels(ImmutableSet.of("x")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x")); } catch (IOException e) { assertNodeLabelsDisabledErrorMessage(e); caught = true; @@ -392,7 +400,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testLabelsToNodes() throws IOException { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); Map<String, Set<NodeId>> labelsToNodes = mgr.getLabelsToNodes(); assertLabelsToNodesEquals( @@ -453,7 +461,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testLabelsToNodesForSelectedLabels() throws IOException { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.addLabelsToNode( ImmutableMap.of( toNodeId("n1:1"), toSet("p1"), @@ -511,7 +519,7 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { boolean failed = false; // As in YARN-2694, we temporarily disable no more than one label existed in // one host - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); try { mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2"))); } catch (IOException e) { @@ -539,29 +547,11 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { failed); } - @Test (timeout = 5000) - public void testUpdateNodeLabels() throws Exception { - boolean failed = false; - - // should fail: label isn't exist - try { - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance( - "p1", false))); - } catch (YarnException e) { - failed = true; + private void verifyNodeLabelAdded(Set<String> expectedAddedLabelNames, + Collection<NodeLabel> addedNodeLabels) { + Assert.assertEquals(expectedAddedLabelNames.size(), addedNodeLabels.size()); + for (NodeLabel label : addedNodeLabels) { + Assert.assertTrue(expectedAddedLabelNames.contains(label.getName())); } - Assert.assertTrue("Should fail since the node label doesn't exist", failed); - - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - - mgr.updateNodeLabels(Arrays.asList( - NodeLabel.newInstance("p1", false), NodeLabel.newInstance("p2", true))); - Assert.assertEquals("p1", mgr.lastUpdatedNodeLabels.get(0).getNodeLabel()); - Assert.assertFalse(mgr.lastUpdatedNodeLabels.get(0).getIsExclusive()); - Assert.assertTrue(mgr.lastUpdatedNodeLabels.get(1).getIsExclusive()); - - // Check exclusive for p1/p2 - Assert.assertFalse(mgr.isExclusiveNodeLabel("p1")); - Assert.assertTrue(mgr.isExclusiveNodeLabel("p2")); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java index 6694290..f070c20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java @@ -84,9 +84,9 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 10000) public void testRecoverWithMirror() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - mgr.addToCluserNodeLabels(toSet("p4")); - mgr.addToCluserNodeLabels(toSet("p5", "p6")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6")); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n3"), toSet("p3"), @@ -110,8 +110,8 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr.init(conf); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -129,8 +129,8 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr.init(conf); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -147,9 +147,9 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 10000) public void testEditlogRecover() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - mgr.addToCluserNodeLabels(toSet("p4")); - mgr.addToCluserNodeLabels(toSet("p5", "p6")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p5", "p6")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n3"), toSet("p3"), @@ -173,8 +173,8 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr.init(conf); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -191,9 +191,12 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test (timeout = 10000) public void testSerilizationAfterRecovery() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); - mgr.addToCluserNodeLabels(toSet("p4")); - mgr.addToCluserNodeLabels(toSet("p5", "p6")); + // Add to cluster node labels, p2/p6 are non-exclusive. + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", true), + NodeLabel.newInstance("p2", false), NodeLabel.newInstance("p3", true), + NodeLabel.newInstance("p4", true), NodeLabel.newInstance("p5", true), + NodeLabel.newInstance("p6", false))); + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"))); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n3"), toSet("p3"), @@ -220,13 +223,6 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { * p6: n6, n7 */ - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("p2", false))); - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("p6", false))); - - /* - * Set p2/p6 to be exclusive - */ - // shutdown mgr and start a new mgr mgr.stop(); @@ -235,8 +231,8 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr.start(); // check variables - Assert.assertEquals(3, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6"))); assertMapContains(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n2"), @@ -258,7 +254,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr = new MockNodeLabelManager(); mgr.init(conf); mgr.start(); - mgr.addToCluserNodeLabels(toSet("p7", "p8")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p7", "p8")); mgr.stop(); /* @@ -267,7 +263,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr = new MockNodeLabelManager(); mgr.init(conf); mgr.start(); - mgr.addToCluserNodeLabels(toSet("p9")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p9")); mgr.stop(); /* @@ -278,8 +274,8 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase { mgr.start(); // check variables - Assert.assertEquals(6, mgr.getClusterNodeLabels().size()); - Assert.assertTrue(mgr.getClusterNodeLabels().containsAll( + Assert.assertEquals(6, mgr.getClusterNodeLabelNames().size()); + Assert.assertTrue(mgr.getClusterNodeLabelNames().containsAll( Arrays.asList("p2", "p4", "p6", "p7", "p8", "p9"))); mgr.stop(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index e4ed43b..c921326 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -80,8 +80,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLa import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsRequest; -import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeLabelsResponse; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; @@ -678,28 +676,6 @@ public class AdminService extends CompositeService implements throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); } } - - @Override - public UpdateNodeLabelsResponse updateNodeLabels( - UpdateNodeLabelsRequest request) throws YarnException, IOException { - String argName = "updateNodeLabels"; - final String msg = "update node labels"; - UserGroupInformation user = checkAcls(argName); - - checkRMStatus(user.getShortUserName(), argName, msg); - - UpdateNodeLabelsResponse response = UpdateNodeLabelsResponse.newInstance(); - - try { - rmContext.getNodeLabelManager().updateNodeLabels( - request.getNodeLabels()); - RMAuditLogger - .logSuccess(user.getShortUserName(), argName, "AdminService"); - return response; - } catch (YarnException ioe) { - throw logAndWrapException(ioe, user.getShortUserName(), argName, msg); - } - } private void checkRMStatus(String user, String argName, String msg) throws StandbyException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 8f9362e..0cc7589 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -135,7 +135,7 @@ class CSQueueUtils { accessibleLabels.addAll(labels); } if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) { - accessibleLabels.addAll(mgr.getClusterNodeLabels()); + accessibleLabels.addAll(mgr.getClusterNodeLabelNames()); } accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 53142b5..97a5d27 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -147,7 +147,7 @@ public class ParentQueue extends AbstractCSQueue { " for children of queue " + queueName); } // check label capacities - for (String nodeLabel : labelManager.getClusterNodeLabels()) { + for (String nodeLabel : labelManager.getClusterNodeLabelNames()) { float capacityByLabel = queueCapacities.getCapacity(nodeLabel); // check children's labels float sum = 0; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index fae0806..6cd6d56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -857,7 +857,7 @@ public class RMWebServices { NodeLabelsInfo ret = new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager() - .getClusterNodeLabels()); + .getClusterNodeLabelNames()); return ret; } @@ -883,7 +883,7 @@ public class RMWebServices { } rm.getRMContext().getNodeLabelManager() - .addToCluserNodeLabels(new HashSet<String>( + .addToCluserNodeLabelsWithDefaultExclusivity(new HashSet<String>( newNodeLabels.getNodeLabels())); return Response.status(Status.OK).build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 430763c..7be982b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -99,6 +99,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.QueueACL; @@ -200,7 +201,7 @@ public class TestClientRMService { }; rm.start(); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); // Add a healthy node with label = x MockNM node = rm.registerNode("host1:1234", 1024); @@ -1407,7 +1408,7 @@ public class TestClientRMService { }; rm.start(); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>(); map.put(NodeId.newInstance("host1", 0), ImmutableSet.of("x")); @@ -1427,7 +1428,7 @@ public class TestClientRMService { GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); Assert.assertTrue(response.getNodeLabels().containsAll( - Arrays.asList("x", "y"))); + Arrays.asList(NodeLabel.newInstance("x"), NodeLabel.newInstance("y")))); // Get node labels mapping GetNodesToLabelsResponse response1 = @@ -1457,7 +1458,7 @@ public class TestClientRMService { }; rm.start(); RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager(); - labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z")); + labelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>(); map.put(NodeId.newInstance("host1", 0), ImmutableSet.of("x")); @@ -1480,7 +1481,8 @@ public class TestClientRMService { GetClusterNodeLabelsResponse response = client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance()); Assert.assertTrue(response.getNodeLabels().containsAll( - Arrays.asList("x", "y", "z"))); + Arrays.asList(NodeLabel.newInstance("x"), NodeLabel.newInstance("y"), + NodeLabel.newInstance("z")))); // Get labels to nodes mapping GetLabelsToNodesResponse response1 = http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index 06a8830..c889446 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -2104,7 +2104,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { clusterNodeLabels.add("y"); clusterNodeLabels.add("z"); // Add node label x,y,z - nodeLabelManager.addToCluserNodeLabels(clusterNodeLabels); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(clusterNodeLabels); // Add node Label to Node h1->x NodeId n1 = NodeId.newInstance("h1", 0); @@ -2129,7 +2129,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { } Assert.assertEquals(clusterNodeLabels.size(), nodeLabelManager - .getClusterNodeLabels().size()); + .getClusterNodeLabelNames().size()); Map<NodeId, Set<String>> nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); @@ -2148,7 +2148,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { nodeLabelManager = rm2.getRMContext().getNodeLabelManager(); Assert.assertEquals(clusterNodeLabels.size(), nodeLabelManager - .getClusterNodeLabels().size()); + .getClusterNodeLabelNames().size()); nodeLabels = nodeLabelManager.getNodeLabels(); Assert.assertEquals(1, nodeLabelManager.getNodeLabels().size()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index 18d7df4..cc5f464 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -337,7 +337,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -386,7 +386,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("X", "Y", "Z")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("X", "Y", "Z")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -439,7 +439,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("X", "Y", "Z")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("X", "Y", "Z")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -491,7 +491,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { }; rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -551,7 +551,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.start(); // adding valid labels try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); @@ -636,7 +636,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { rm.start(); try { - nodeLabelsMgr.addToCluserNodeLabels(toSet("A", "B", "C")); + nodeLabelsMgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("A", "B", "C")); } catch (IOException e) { Assert.fail("Caught Exception while intializing"); e.printStackTrace(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java index 14d96a0..9548029 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java @@ -57,7 +57,8 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager { } @Override - public void storeNewClusterNodeLabels(Set<String> label) throws IOException { + public void storeNewClusterNodeLabels(List<NodeLabel> label) + throws IOException { // do nothing } @@ -65,12 +66,6 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager { public void close() throws IOException { // do nothing } - - @Override - public void updateNodeLabels(List<NodeLabel> updatedNodeLabels) - throws IOException { - // do nothing - } }; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index ace8742..05bb1e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -63,7 +63,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testGetLabelResourceWhenNodeActiveDeactive() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); @@ -81,7 +81,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { // check add labels multiple times shouldn't overwrite // original attributes on labels like resource - mgr.addToCluserNodeLabels(toSet("p1", "p4")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p4")); Assert.assertEquals(mgr.getResourceByLabel("p1", null), Resources.add(SMALL_RESOURCE, LARGE_NODE)); Assert.assertEquals(mgr.getResourceByLabel("p4", null), EMPTY_RESOURCE); @@ -120,7 +120,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { @SuppressWarnings({ "unchecked", "rawtypes" }) @Test(timeout = 5000) public void testGetLabelResource() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); @@ -137,7 +137,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { Assert.assertEquals(mgr.getResourceByLabel("p3", null), SMALL_RESOURCE); // add more labels - mgr.addToCluserNodeLabels(toSet("p4", "p5", "p6")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p4", "p5", "p6")); mgr.replaceLabelsOnNode((Map) ImmutableMap.of(toNodeId("n4"), toSet("p1"), toNodeId("n5"), toSet("p2"), toNodeId("n6"), toSet("p3"), toNodeId("n7"), toSet("p4"), toNodeId("n8"), toSet("p5"))); @@ -216,7 +216,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { * host3 : yellow * host4 : */ - mgr.addToCluserNodeLabels(toSet("red", "blue", "yellow")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("red", "blue", "yellow")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow"))); @@ -397,7 +397,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { Resources.multiply(SMALL_RESOURCE, 4)); // change two of these nodes to p1, check resource of no_label and P1 - mgr.addToCluserNodeLabels(toSet("p1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1")); mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"), toSet("p1"))); @@ -412,7 +412,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testRemoveLabelsFromNode() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3"))); // active one NM to n1:1 @@ -432,7 +432,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testGetLabelsOnNodesWhenNodeActiveDeactive() throws Exception { - mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1", "p2", "p3")); mgr.replaceLabelsOnNode(ImmutableMap.of( toNodeId("n1"), toSet("p2"))); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"))); @@ -489,7 +489,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { @Test(timeout = 5000) public void testPullRMNodeLabelsInfo() throws IOException { - mgr.addToCluserNodeLabels(toSet("x", "y", "z")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("x", "y", "z")); mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); @@ -516,7 +516,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels())); // Add labels and replace labels on node - mgr.addToCluserNodeLabels(toSet("p1")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("p1")); mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); // p1 -> n1, n1:1 Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 4aa818b..1a34e57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -2578,7 +2578,7 @@ public class TestCapacityScheduler { final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.init(conf); - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); MemoryRMStateStore memStore = new MemoryRMStateStore(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index 1e339d9..c5439d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -98,7 +98,7 @@ public class TestCapacitySchedulerNodeLabelUpdate { @Test (timeout = 30000) public void testNodeUpdate() throws Exception { // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); // set mapping: // h1 -> x http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index 47398e3..6aceda5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; import java.util.List; import java.util.Set; @@ -34,7 +32,6 @@ import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; @@ -54,13 +51,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState; -import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Assert; http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java index 5155db5..46aa7ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java @@ -143,7 +143,7 @@ public class TestNodeLabelContainerAllocation { mgr.init(conf); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); @@ -226,7 +226,7 @@ public class TestNodeLabelContainerAllocation { */ // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"), NodeId.newInstance("h3", 0), toSet("y"), NodeId.newInstance("h4", 0), @@ -310,7 +310,7 @@ public class TestNodeLabelContainerAllocation { @Test (timeout = 120000) public void testContainerAllocateWithLabels() throws Exception { // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); @@ -386,7 +386,7 @@ public class TestNodeLabelContainerAllocation { // instead, it uses default queue label expression // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); @@ -483,9 +483,8 @@ public class TestNodeLabelContainerAllocation { */ // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); - // Makes y to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("y", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x"), NodeLabel.newInstance("y", false))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("y"))); // inject node label manager @@ -561,9 +560,9 @@ public class TestNodeLabelContainerAllocation { */ // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x"), NodeLabel.newInstance("y", false))); // Makes y to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("y", false))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("y"))); // inject node label manager @@ -612,10 +611,8 @@ public class TestNodeLabelContainerAllocation { * non-labeled request should get allocation on non-labeled nodes first. */ - // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false), NodeLabel.newInstance("y"))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager @@ -734,9 +731,8 @@ public class TestNodeLabelContainerAllocation { csConf.setCapacityByLabel(C2, "x", 0); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false), NodeLabel.newInstance("y"))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager @@ -844,9 +840,8 @@ public class TestNodeLabelContainerAllocation { csConf.setUserLimitFactor(B, 5); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false), NodeLabel.newInstance("y"))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager @@ -909,9 +904,8 @@ public class TestNodeLabelContainerAllocation { */ // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false), NodeLabel.newInstance("y"))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager @@ -984,9 +978,8 @@ public class TestNodeLabelContainerAllocation { csConf.setMaximumCapacityByLabel(B, "x", 50); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager @@ -1114,9 +1107,8 @@ public class TestNodeLabelContainerAllocation { csConf.setCapacityByLabel(B, "x", 50); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager @@ -1349,9 +1341,8 @@ public class TestNodeLabelContainerAllocation { csConf.setCapacity(D, 25); // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x")); - // Makes x to be non-exclusive node labels - mgr.updateNodeLabels(Arrays.asList(NodeLabel.newInstance("x", false))); + mgr.addToCluserNodeLabels(ImmutableSet.of( + NodeLabel.newInstance("x", false))); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"))); // inject node label manager http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 34f7c2c..0206772 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -417,7 +417,7 @@ public class TestQueueParsing { @Test public void testQueueParsingReinitializeWithLabels() throws IOException { - nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithoutLabels(csConf); @@ -505,7 +505,7 @@ public class TestQueueParsing { @Test public void testQueueParsingWithLabels() throws IOException { - nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = @@ -529,7 +529,7 @@ public class TestQueueParsing { @Test public void testQueueParsingWithLabelsInherit() throws IOException { - nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("red", "blue")); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = @@ -668,7 +668,7 @@ public class TestQueueParsing { // Initialize a cluster with labels, but doesn't use them, reinitialize // shouldn't fail - nodeLabelManager.addToCluserNodeLabels(labels); + nodeLabelManager.addToCluserNodeLabelsWithDefaultExclusivity(labels); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ec6e7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java index fc9e14a..eeec940 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestWorkPreservingRMRestartForNodeLabel.java @@ -129,7 +129,7 @@ public class TestWorkPreservingRMRestartForNodeLabel { // instead, it uses default queue label expression // set node -> label - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); @@ -205,7 +205,7 @@ public class TestWorkPreservingRMRestartForNodeLabel { // Re-start RM mgr = new NullRMNodeLabelsManager(); mgr.init(conf); - mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y")); + mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y")); mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x"), NodeId.newInstance("h2", 0), toSet("y"))); MockRM rm2 =
