http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index 9f3e925..682d6ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.yarn.api; import java.io.IOException; +import java.util.Arrays; import org.apache.commons.lang3.Range; import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto; @@ -112,7 +113,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestP import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl; -import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; import org.apache.hadoop.yarn.api.records.ApplicationId; @@ -120,6 +120,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeout; +import org.apache.hadoop.yarn.api.records.CollectorInfo; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; @@ -131,9 +132,12 @@ import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.NMToken; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.PreemptionContainer; import org.apache.hadoop.yarn.api.records.PreemptionContract; import org.apache.hadoop.yarn.api.records.PreemptionMessage; @@ -152,8 +156,8 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest; import org.apache.hadoop.yarn.api.records.ReservationRequests; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceAllocationRequest; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.ResourceSizing; @@ -183,10 +187,13 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ExecutionTypeRequestPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeInfoPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeToAttributeValuePBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionContainerPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionContractPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl; @@ -222,10 +229,14 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryContextProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributeValueProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProto; import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionContainerProto; import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionContractProto; import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionMessageProto; @@ -243,7 +254,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto; import org.apache.hadoop.yarn.proto.YarnProtos.StrictPreemptionContractProto; import org.apache.hadoop.yarn.proto.YarnProtos.URLProto; import org.apache.hadoop.yarn.proto.YarnProtos.YarnClusterMetricsProto; -import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto; import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto; @@ -274,6 +284,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto; @@ -304,6 +315,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto; @@ -328,9 +341,6 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl; @@ -366,7 +376,6 @@ import org.junit.Ignore; import org.junit.Test; import com.google.common.collect.ImmutableSet; -import java.util.Arrays; /** * Test class for YARN API protocol records. @@ -450,10 +459,12 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { generateByNewInstance(SchedulingRequest.class); generateByNewInstance(RejectedSchedulingRequest.class); //for Node attribute support + generateByNewInstance(NodeAttributeKey.class); generateByNewInstance(NodeAttribute.class); generateByNewInstance(NodeToAttributes.class); + generateByNewInstance(NodeToAttributeValue.class); + generateByNewInstance(NodeAttributeInfo.class); generateByNewInstance(NodesToAttributesMappingRequest.class); - } @Test @@ -1250,11 +1261,29 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { } @Test + public void testNodeAttributeKeyPBImpl() throws Exception { + validatePBImplRecord(NodeAttributeKeyPBImpl.class, + NodeAttributeKeyProto.class); + } + + @Test + public void testNodeToAttributeValuePBImpl() throws Exception { + validatePBImplRecord(NodeToAttributeValuePBImpl.class, + NodeToAttributeValueProto.class); + } + + @Test public void testNodeAttributePBImpl() throws Exception { validatePBImplRecord(NodeAttributePBImpl.class, NodeAttributeProto.class); } @Test + public void testNodeAttributeInfoPBImpl() throws Exception { + validatePBImplRecord(NodeAttributeInfoPBImpl.class, + NodeAttributeInfoProto.class); + } + + @Test public void testNodeToAttributesPBImpl() throws Exception { validatePBImplRecord(NodeToAttributesPBImpl.class, NodeToAttributesProto.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java index d4384b4..bad74d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.BeforeClass; @@ -120,7 +121,8 @@ public class TestConfigurationNodeAttributesProvider { while(times>0) { Set<NodeAttribute> current = spyProvider.getDescriptors(); Assert.assertEquals(1, current.size()); - String attributeName = current.iterator().next().getAttributeName(); + String attributeName = + current.iterator().next().getAttributeKey().getAttributeName(); if ("host".equals(attributeName)){ numOfOldValue++; } else if ("os".equals(attributeName)) { @@ -173,7 +175,7 @@ public class TestConfigurationNodeAttributesProvider { GenericTestUtils.waitFor(() -> { Set<NodeAttribute> attributes = spyProvider.getDescriptors(); return "os".equalsIgnoreCase(attributes - .iterator().next().getAttributeName()); + .iterator().next().getAttributeKey().getAttributeName()); }, 500, 1000); } catch (Exception e) { // Make sure we get the timeout exception. @@ -204,21 +206,22 @@ public class TestConfigurationNodeAttributesProvider { Iterator<NodeAttribute> ait = attributes.iterator(); while(ait.hasNext()) { - NodeAttribute at = ait.next(); + NodeAttribute attr = ait.next(); + NodeAttributeKey at = attr.getAttributeKey(); if (at.getAttributeName().equals("hostname")) { Assert.assertEquals("hostname", at.getAttributeName()); Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED, at.getAttributePrefix()); Assert.assertEquals(NodeAttributeType.STRING, - at.getAttributeType()); - Assert.assertEquals("host1234", at.getAttributeValue()); + attr.getAttributeType()); + Assert.assertEquals("host1234", attr.getAttributeValue()); } else if (at.getAttributeName().equals("uptime")) { Assert.assertEquals("uptime", at.getAttributeName()); Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED, at.getAttributePrefix()); Assert.assertEquals(NodeAttributeType.STRING, - at.getAttributeType()); - Assert.assertEquals("321543", at.getAttributeValue()); + attr.getAttributeType()); + Assert.assertEquals("321543", attr.getAttributeValue()); } else { Assert.fail("Unexpected attribute"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java index f764626..3e2e161 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java @@ -129,7 +129,7 @@ public class TestScriptBasedNodeAttributesProvider { .getDescriptors().iterator(); while (it.hasNext()) { NodeAttribute att = it.next(); - switch (att.getAttributeName()) { + switch (att.getAttributeKey().getAttributeName()) { case "host": Assert.assertEquals(NodeAttributeType.STRING, att.getAttributeType()); Assert.assertEquals("host1234", att.getAttributeValue()); @@ -143,7 +143,8 @@ public class TestScriptBasedNodeAttributesProvider { Assert.assertEquals("10.0.0.1", att.getAttributeValue()); break; default: - Assert.fail("Unexpected attribute name " + att.getAttributeName()); + Assert.fail("Unexpected attribute name " + + att.getAttributeKey().getAttributeName()); break; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 12f8aaf..77f8f66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -1035,7 +1035,7 @@ public class AdminService extends CompositeService implements List<NodeAttribute> nodeAttributes = nodeToAttributes.getNodeAttributes(); if (!nodeAttributes.stream() .allMatch(nodeAttribute -> NodeAttribute.PREFIX_CENTRALIZED - .equals(nodeAttribute.getAttributePrefix()))) { + .equals(nodeAttribute.getAttributeKey().getAttributePrefix()))) { throw new IOException("Invalid Attribute Mapping for the node " + node + ". Prefix should be " + NodeAttribute.PREFIX_CENTRALIZED); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 3f24355..ad796f6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -29,12 +29,14 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; import org.apache.commons.cli.UnrecognizedOptionException; import org.apache.commons.lang3.Range; @@ -134,8 +136,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueInfo; @@ -155,6 +160,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.nodelabels.AttributeValue; import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; @@ -1851,9 +1857,23 @@ public class ClientRMService extends AbstractService implements GetAttributesToNodesRequest request) throws YarnException, IOException { NodeAttributesManager attributesManager = rmContext.getNodeAttributesManager(); - GetAttributesToNodesResponse response = GetAttributesToNodesResponse - .newInstance(attributesManager - .getAttributesToNodes(request.getNodeAttributes())); + Map<NodeAttributeKey, List<NodeToAttributeValue>> attrToNodesWithStrVal = + new HashMap<>(); + Map<NodeAttributeKey, Map<String, AttributeValue>> attributesToNodes = + attributesManager.getAttributesToNodes(request.getNodeAttributes()); + for (Map.Entry<NodeAttributeKey, Map<String, AttributeValue>> attrib : + attributesToNodes.entrySet()) { + Map<String, AttributeValue> nodesToVal = attrib.getValue(); + List<NodeToAttributeValue> nodeToAttrValList = new ArrayList<>(); + for (Map.Entry<String, AttributeValue> nodeToVal : nodesToVal + .entrySet()) { + nodeToAttrValList.add(NodeToAttributeValue + .newInstance(nodeToVal.getKey(), nodeToVal.getValue().getValue())); + } + attrToNodesWithStrVal.put(attrib.getKey(), nodeToAttrValList); + } + GetAttributesToNodesResponse response = + GetAttributesToNodesResponse.newInstance(attrToNodesWithStrVal); return response; } @@ -1865,8 +1885,11 @@ public class ClientRMService extends AbstractService implements rmContext.getNodeAttributesManager(); Set<NodeAttribute> attributes = attributesManager.getClusterNodeAttributes(null); + GetClusterNodeAttributesResponse response = - GetClusterNodeAttributesResponse.newInstance(attributes); + GetClusterNodeAttributesResponse.newInstance( + attributes.stream().map(attr -> NodeAttributeInfo.newInstance(attr)) + .collect(Collectors.toSet())); return response; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java index 4f4400f..cbb5ecf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java @@ -660,7 +660,7 @@ public class ResourceTrackerService extends AbstractService implements // Validate attributes if (!nodeAttributes.stream().allMatch( nodeAttribute -> NodeAttribute.PREFIX_DISTRIBUTED - .equals(nodeAttribute.getAttributePrefix()))) { + .equals(nodeAttribute.getAttributeKey().getAttributePrefix()))) { // All attributes must be in same prefix: nm.yarn.io. // Since we have the checks in NM to make sure attributes reported // in HB are with correct prefix, so it should not reach here. http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java index 1645602..6b176d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java @@ -19,21 +19,20 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentHashMap.KeySetView; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; -import java.util.ArrayList; -import java.util.List; import com.google.common.base.Strings; import org.apache.commons.lang3.StringUtils; @@ -42,6 +41,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.nodelabels.StringAttributeValue; import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes; +import com.google.common.base.Strings; + /** * Manager holding the attributes to Labels. */ @@ -75,8 +77,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { // TODO may be we can have a better collection here. // this will be updated to get the attributeName to NM mapping - private ConcurrentHashMap<NodeAttribute, RMNodeAttribute> clusterAttributes = - new ConcurrentHashMap<>(); + private ConcurrentHashMap<NodeAttributeKey, RMNodeAttribute> clusterAttributes + = new ConcurrentHashMap<>(); // hostname -> (Map (attributeName -> NodeAttribute)) // Instead of NodeAttribute, plan to have it in future as AttributeValue @@ -149,7 +151,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { private void internalUpdateAttributesOnNodes( Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping, AttributeMappingOperationType op, - Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded, + Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded, String attributePrefix) { try { writeLock.lock(); @@ -210,13 +212,14 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { private void removeNodeFromAttributes(String nodeHost, Set<NodeAttribute> attributeMappings) { for (NodeAttribute rmAttribute : attributeMappings) { - RMNodeAttribute host = clusterAttributes.get(rmAttribute); + RMNodeAttribute host = + clusterAttributes.get(rmAttribute.getAttributeKey()); if (host != null) { host.removeNode(nodeHost); // If there is no other host has such attribute, // remove it from the global mapping. if (host.getAssociatedNodeIds().isEmpty()) { - clusterAttributes.remove(rmAttribute); + clusterAttributes.remove(rmAttribute.getAttributeKey()); } } } @@ -224,12 +227,16 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { private void addNodeToAttribute(String nodeHost, Map<NodeAttribute, AttributeValue> attributeMappings) { - for (NodeAttribute attribute : attributeMappings.keySet()) { - RMNodeAttribute rmNodeAttribute = clusterAttributes.get(attribute); + for (Entry<NodeAttribute, AttributeValue> attributeEntry : attributeMappings + .entrySet()) { + + RMNodeAttribute rmNodeAttribute = + clusterAttributes.get(attributeEntry.getKey().getAttributeKey()); if (rmNodeAttribute != null) { - rmNodeAttribute.addNode(nodeHost); + rmNodeAttribute.addNode(nodeHost, attributeEntry.getValue()); } else { - clusterAttributes.put(attribute, new RMNodeAttribute(attribute)); + clusterAttributes.put(attributeEntry.getKey().getAttributeKey(), + new RMNodeAttribute(attributeEntry.getKey())); } } } @@ -257,7 +264,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { */ protected Map<String, Map<NodeAttribute, AttributeValue>> validate( Map<String, Set<NodeAttribute>> nodeAttributeMapping, - Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded, + Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded, boolean isRemoveOperation) throws IOException { Map<String, Map<NodeAttribute, AttributeValue>> nodeToAttributesMap = new TreeMap<>(); @@ -274,19 +281,21 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { // validate for attributes for (NodeAttribute attribute : nodeToAttrMappingEntry.getValue()) { - String attributeName = attribute.getAttributeName().trim(); + NodeAttributeKey attributeKey = attribute.getAttributeKey(); + String attributeName = attributeKey.getAttributeName().trim(); NodeLabelUtil.checkAndThrowLabelName(attributeName); NodeLabelUtil - .checkAndThrowAttributePrefix(attribute.getAttributePrefix()); + .checkAndThrowAttributePrefix(attributeKey.getAttributePrefix()); // ensure trimmed values are set back - attribute.setAttributeName(attributeName); - attribute.setAttributePrefix(attribute.getAttributePrefix().trim()); + attributeKey.setAttributeName(attributeName); + attributeKey + .setAttributePrefix(attributeKey.getAttributePrefix().trim()); // verify for type against prefix/attributeName if (validateForAttributeTypeMismatch(isRemoveOperation, attribute, newAttributesToBeAdded)) { - newAttributesToBeAdded.put(attribute, + newAttributesToBeAdded.put(attribute.getAttributeKey(), new RMNodeAttribute(attribute)); } // TODO type based value setting needs to be done using a factory @@ -310,9 +319,11 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { */ private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation, NodeAttribute attribute, - Map<NodeAttribute, RMNodeAttribute> newAttributes) + Map<NodeAttributeKey, RMNodeAttribute> newAttributes) throws IOException { - if (isRemoveOperation && !clusterAttributes.containsKey(attribute)) { + NodeAttributeKey attributeKey = attribute.getAttributeKey(); + if (isRemoveOperation + && !clusterAttributes.containsKey(attributeKey)) { // no need to validate anything as its remove operation and attribute // doesn't exist. return false; // no need to add as its remove operation @@ -320,10 +331,10 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { // already existing or attribute is mapped to another Node in the // current command, then check whether the attribute type is matching NodeAttribute existingAttribute = - (clusterAttributes.containsKey((attribute)) - ? clusterAttributes.get(attribute).getAttribute() - : (newAttributes.containsKey(attribute) - ? newAttributes.get(attribute).getAttribute() + (clusterAttributes.containsKey(attributeKey) + ? clusterAttributes.get(attributeKey).getAttribute() + : (newAttributes.containsKey(attributeKey) + ? newAttributes.get(attributeKey).getAttribute() : null)); if (existingAttribute == null) { return true; @@ -331,7 +342,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { .getAttributeType()) { throw new IOException("Attribute name - type is not matching with " + "already configured mapping for the attribute " - + attribute.getAttributeName() + " existing : " + + attributeKey + " existing : " + existingAttribute.getAttributeType() + ", new :" + attribute.getAttributeType()); } @@ -347,37 +358,39 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { } @Override - public Set<NodeAttribute> getClusterNodeAttributes(Set<String> prefix) { + public Set<NodeAttribute> getClusterNodeAttributes( + Set<String> prefix) { Set<NodeAttribute> attributes = new HashSet<>(); - KeySetView<NodeAttribute, RMNodeAttribute> allAttributes = - clusterAttributes.keySet(); + Set<Entry<NodeAttributeKey, RMNodeAttribute>> allAttributes = + clusterAttributes.entrySet(); // Return all if prefix is not given. - if (prefix == null || prefix.isEmpty()) { - attributes.addAll(allAttributes); - return attributes; - } + boolean forAllPrefix = prefix == null || prefix.isEmpty(); // Try search attributes by prefix and return valid ones. - Iterator<NodeAttribute> iterator = allAttributes.iterator(); + Iterator<Entry<NodeAttributeKey, RMNodeAttribute>> iterator = + allAttributes.iterator(); while (iterator.hasNext()) { - NodeAttribute current = iterator.next(); - if (prefix.contains(current.getAttributePrefix())) { - attributes.add(current); + Entry<NodeAttributeKey, RMNodeAttribute> current = iterator.next(); + NodeAttributeKey attrID = current.getKey(); + RMNodeAttribute rmAttr = current.getValue(); + if (forAllPrefix || prefix.contains(attrID.getAttributePrefix())) { + attributes.add(rmAttr.getAttribute()); } } return attributes; } @Override - public Map<NodeAttribute, Set<String>> getAttributesToNodes( - Set<NodeAttribute> attributes) { + public Map<NodeAttributeKey, Map<String, AttributeValue>> getAttributesToNodes( + Set<NodeAttributeKey> attributes) { try { readLock.lock(); boolean fetchAllAttributes = (attributes == null || attributes.isEmpty()); - Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>(); - for (Entry<NodeAttribute, RMNodeAttribute> attributeEntry : + Map<NodeAttributeKey, Map<String, AttributeValue>> attributesToNodes = + new HashMap<>(); + for (Entry<NodeAttributeKey, RMNodeAttribute> attributeEntry : clusterAttributes.entrySet()) { - if (fetchAllAttributes || attributes - .contains(attributeEntry.getKey())) { + if (fetchAllAttributes + || attributes.contains(attributeEntry.getKey())) { attributesToNodes.put(attributeEntry.getKey(), attributeEntry.getValue().getAssociatedNodeIds()); } @@ -391,8 +404,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { public Resource getResourceByAttribute(NodeAttribute attribute) { try { readLock.lock(); - return clusterAttributes.containsKey(attribute) - ? clusterAttributes.get(attribute).getResource() + return clusterAttributes.containsKey(attribute.getAttributeKey()) + ? clusterAttributes.get(attribute.getAttributeKey()).getResource() : Resource.newInstance(0, 0); } finally { readLock.unlock(); @@ -425,7 +438,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { attrs = new ArrayList<>(); for (Entry<NodeAttribute, AttributeValue> nodeAttr : v.attributes .entrySet()) { - if (prefix.contains(nodeAttr.getKey().getAttributePrefix())) { + if (prefix.contains( + nodeAttr.getKey().getAttributeKey().getAttributePrefix())) { attrs.add(nodeAttr.getKey()); } } @@ -473,7 +487,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { } host.activateNode(resource); for (NodeAttribute attribute : host.getAttributes().keySet()) { - clusterAttributes.get(attribute).removeNode(resource); + clusterAttributes.get(attribute.getAttributeKey()).removeNode(resource); } } finally { writeLock.unlock(); @@ -485,7 +499,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { writeLock.lock(); Host host = nodeCollections.get(nodeId.getHost()); for (NodeAttribute attribute : host.getAttributes().keySet()) { - clusterAttributes.get(attribute).removeNode(host.getResource()); + clusterAttributes.get(attribute.getAttributeKey()) + .removeNode(host.getResource()); } host.deactivateNode(); } finally { @@ -531,7 +546,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { this.attributes.entrySet().iterator(); while (it.hasNext()) { Entry<NodeAttribute, AttributeValue> current = it.next(); - if (prefix.equals(current.getKey().getAttributePrefix())) { + if (prefix.equals( + current.getKey().getAttributeKey().getAttributePrefix())) { it.remove(); } } @@ -659,7 +675,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { Map<String, Set<NodeAttribute>> nodeAttributeMapping, AttributeMappingOperationType mappingType, String attributePrefix) throws IOException { - Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded = + Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded = new HashMap<>(); Map<String, Map<NodeAttribute, AttributeValue>> validMapping = validate(nodeAttributeMapping, newAttributesToBeAdded, false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java index 1645d13..93b901e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java @@ -21,9 +21,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeLabel; /** @@ -56,4 +58,21 @@ public final class NodeLabelsUtils { throw new IOException(msg); } } + + /** + * Returns a set of node attributes whose name exists in the provided + * <code>attributeNames</code> list. + * + * @param attributeNames For this given list of attribute names get the + * cluster NodeAttributes + * @param clusterNodeAttributes set of node Attributes + * @return set of Node Attributes which maps to the give attributes names + */ + public static Set <NodeAttribute> getNodeAttributesByName( + Set<String> attributeNames, Set<NodeAttribute> clusterNodeAttributes) { + return clusterNodeAttributes.stream() + .filter(attribute -> attributeNames + .contains(attribute.getAttributeKey().getAttributeName())) + .collect(Collectors.toSet()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java index bbc2ec3..8384312 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java @@ -41,8 +41,8 @@ public class NodeAttributeInfo { } public NodeAttributeInfo(NodeAttribute nodeAttribute) { - this.prefix = nodeAttribute.getAttributePrefix(); - this.name = nodeAttribute.getAttributeName(); + this.prefix = nodeAttribute.getAttributeKey().getAttributePrefix(); + this.name = nodeAttribute.getAttributeKey().getAttributeName(); this.type = nodeAttribute.getAttributeType().toString(); this.value = nodeAttribute.getAttributeValue(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 95ad35a..39892f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -18,16 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager; -import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; - -import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; -import org.apache.hadoop.yarn.api.records.NodeAttribute; -import org.apache.hadoop.yarn.api.records.NodeAttributeType; -import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -78,6 +68,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -88,6 +82,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; @@ -117,10 +113,15 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeInfo; +import org.apache.hadoop.yarn.api.records.NodeAttributeKey; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.NodeToAttributeValue; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueConfigurations; @@ -142,6 +143,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter; import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; @@ -163,7 +165,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager; import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager; - import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; import org.apache.hadoop.yarn.server.security.ApplicationACLsManager; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -176,11 +177,11 @@ import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; public class TestClientRMService { @@ -2048,11 +2049,12 @@ public class TestClientRMService { GetClusterNodeAttributesRequest.newInstance(); GetClusterNodeAttributesResponse response = client.getClusterNodeAttributes(request); - Set<NodeAttribute> attributes = response.getNodeAttributes(); + Set<NodeAttributeInfo> attributes = response.getNodeAttributes(); Assert.assertEquals("Size not correct", 3, attributes.size()); - Assert.assertTrue(attributes.contains(gpu)); - Assert.assertTrue(attributes.contains(os)); - Assert.assertTrue(attributes.contains(docker)); + Assert.assertTrue(attributes.contains(NodeAttributeInfo.newInstance(gpu))); + Assert.assertTrue(attributes.contains(NodeAttributeInfo.newInstance(os))); + Assert + .assertTrue(attributes.contains(NodeAttributeInfo.newInstance(docker))); rpc.stopProxy(client, conf); rm.close(); } @@ -2071,17 +2073,17 @@ public class TestClientRMService { NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager(); String node1 = "host1"; String node2 = "host2"; - NodeAttribute gpu = NodeAttribute - .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", - NodeAttributeType.STRING, "nvida"); - NodeAttribute os = NodeAttribute - .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttribute gpu = + NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvidia"); + NodeAttribute os = + NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", NodeAttributeType.STRING, "windows64"); - NodeAttribute docker = NodeAttribute - .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttribute docker = + NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", NodeAttributeType.STRING, "docker0"); - NodeAttribute dist = NodeAttribute - .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", + NodeAttribute dist = + NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", NodeAttributeType.STRING, "3_0_2"); Map<String, Set<NodeAttribute>> nodes = new HashMap<>(); nodes.put(node1, ImmutableSet.of(gpu, os, dist)); @@ -2099,35 +2101,55 @@ public class TestClientRMService { GetAttributesToNodesRequest.newInstance(); GetAttributesToNodesResponse response = client.getAttributesToNodes(request); - Map<NodeAttribute, Set<String>> attrs = response.getAttributesToNodes(); + Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs = + response.getAttributesToNodes(); Assert.assertEquals(response.getAttributesToNodes().size(), 4); - Assert.assertEquals(attrs.get(dist).size(), 2); - Assert.assertEquals(attrs.get(os).size(), 1); - Assert.assertEquals(attrs.get(gpu).size(), 1); - Assert.assertTrue(attrs.get(dist).contains(node1)); - Assert.assertTrue(attrs.get(dist).contains(node2)); - Assert.assertTrue(attrs.get(docker).contains(node2)); - - GetAttributesToNodesRequest request2 = - GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker)); + Assert.assertEquals(attrs.get(dist.getAttributeKey()).size(), 2); + Assert.assertEquals(attrs.get(os.getAttributeKey()).size(), 1); + Assert.assertEquals(attrs.get(gpu.getAttributeKey()).size(), 1); + Assert.assertTrue(findHostnameAndValInMapping(node1, "3_0_2", + attrs.get(dist.getAttributeKey()))); + Assert.assertTrue(findHostnameAndValInMapping(node2, "3_0_2", + attrs.get(dist.getAttributeKey()))); + Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0", + attrs.get(docker.getAttributeKey()))); + + GetAttributesToNodesRequest request2 = GetAttributesToNodesRequest + .newInstance(ImmutableSet.of(docker.getAttributeKey())); GetAttributesToNodesResponse response2 = client.getAttributesToNodes(request2); - Map<NodeAttribute, Set<String>> attrs2 = response2.getAttributesToNodes(); - Assert.assertEquals(response2.getAttributesToNodes().size(), 1); - Assert.assertTrue(attrs.get(docker).contains(node2)); + Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs2 = + response2.getAttributesToNodes(); + Assert.assertEquals(attrs2.size(), 1); + Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0", + attrs2.get(docker.getAttributeKey()))); GetAttributesToNodesRequest request3 = - GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os)); + GetAttributesToNodesRequest.newInstance( + ImmutableSet.of(docker.getAttributeKey(), os.getAttributeKey())); GetAttributesToNodesResponse response3 = client.getAttributesToNodes(request3); - Map<NodeAttribute, Set<String>> attrs3 = response3.getAttributesToNodes(); - Assert.assertEquals(response3.getAttributesToNodes().size(), 2); - Assert.assertTrue(attrs.get(os).contains(node1)); - Assert.assertTrue(attrs.get(docker).contains(node2)); + Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs3 = + response3.getAttributesToNodes(); + Assert.assertEquals(attrs3.size(), 2); + Assert.assertTrue(findHostnameAndValInMapping(node1, "windows64", + attrs3.get(os.getAttributeKey()))); + Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0", + attrs3.get(docker.getAttributeKey()))); rpc.stopProxy(client, conf); rm.close(); } + private boolean findHostnameAndValInMapping(String hostname, String attrVal, + List<NodeToAttributeValue> mappingVals) { + for (NodeToAttributeValue value : mappingVals) { + if (value.getHostname().equals(hostname)) { + return attrVal.equals(value.getAttributeValue()); + } + } + return false; + } + @Test(timeout = 120000) public void testGetNodesToAttributes() throws IOException, YarnException { MockRM rm = new MockRM() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java index adb7fe0..e40b3c0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java @@ -880,7 +880,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { .getAttributesForNode(nodeId.getHost()); Assert.assertEquals(1, attrs.size()); NodeAttribute na = attrs.keySet().iterator().next(); - Assert.assertEquals("host", na.getAttributeName()); + Assert.assertEquals("host", na.getAttributeKey().getAttributeName()); Assert.assertEquals("host2", na.getAttributeValue()); Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType()); @@ -900,7 +900,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase { attrs = attributeManager.getAttributesForNode(nodeId.getHost()); Assert.assertEquals(1, attrs.size()); na = attrs.keySet().iterator().next(); - Assert.assertEquals("host", na.getAttributeName()); + Assert.assertEquals("host", na.getAttributeKey().getAttributeName()); Assert.assertEquals("host3", na.getAttributeValue()); Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java index e2ee8b4..502f9d0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java @@ -253,8 +253,10 @@ public class TestFileSystemNodeAttributeStore { public void checkNodeAttributeEqual(NodeAttribute atr1, NodeAttribute atr2) { Assert.assertEquals(atr1.getAttributeType(), atr2.getAttributeType()); - Assert.assertEquals(atr1.getAttributeName(), atr2.getAttributeName()); - Assert.assertEquals(atr1.getAttributePrefix(), atr2.getAttributePrefix()); + Assert.assertEquals(atr1.getAttributeKey().getAttributeName(), + atr2.getAttributeKey().getAttributeName()); + Assert.assertEquals(atr1.getAttributeKey().getAttributePrefix(), + atr2.getAttributeKey().getAttributePrefix()); Assert.assertEquals(atr1.getAttributeValue(), atr2.getAttributeValue()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java index b8c5bc9..9bc9388 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java @@ -344,11 +344,12 @@ public class TestNodeAttributesManager { clusterAttributes = attributesManager.getClusterNodeAttributes( Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED)); Assert.assertEquals(1, clusterAttributes.size()); - NodeAttribute att = clusterAttributes.iterator().next(); - Assert.assertEquals("dist-node-attribute-v2_0", att.getAttributeName()); + NodeAttribute attr = clusterAttributes.iterator().next(); + Assert.assertEquals("dist-node-attribute-v2_0", + attr.getAttributeKey().getAttributeName()); Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED, - att.getAttributePrefix()); - Assert.assertEquals("dist_v3_0", att.getAttributeValue()); + attr.getAttributeKey().getAttributePrefix()); + Assert.assertEquals("dist_v3_0", attr.getAttributeValue()); // Replace all attributes toReplaceMap.put(HOSTNAMES[0], http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc230dd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java index ceabe65..3d9b394 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java @@ -849,6 +849,6 @@ public class FederationClientInterceptor @Override public GetNodesToAttributesResponse getNodesToAttributes( GetNodesToAttributesRequest request) throws YarnException, IOException { - throw new NotImplementedException(); + throw new NotImplementedException("Code is not implemented"); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
