YARN-8100. Support API interface to query cluster attributes and attribute to nodes. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/efcca85f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/efcca85f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/efcca85f Branch: refs/heads/YARN-3409 Commit: efcca85f534790a5f9df09c0f2fa6a772b3d3add Parents: 5fc9163 Author: Naganarasimha <naganarasimha...@apache.org> Authored: Tue Apr 10 07:28:53 2018 +0800 Committer: Naganarasimha <naganarasimha...@apache.org> Committed: Tue Apr 10 07:28:53 2018 +0800 ---------------------------------------------------------------------- .../hadoop/mapred/ResourceMgrDelegate.java | 13 ++ .../hadoop/mapred/TestClientRedirect.java | 17 ++ .../yarn/api/ApplicationClientProtocol.java | 37 ++++ .../GetAttributesToNodesRequest.java | 70 +++++++ .../GetAttributesToNodesResponse.java | 62 +++++++ .../GetClusterNodeAttributesRequest.java | 47 +++++ .../GetClusterNodeAttributesResponse.java | 72 ++++++++ .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../main/proto/applicationclient_protocol.proto | 2 + .../src/main/proto/yarn_protos.proto | 4 + .../src/main/proto/yarn_service_protos.proto | 15 ++ .../hadoop/yarn/client/api/YarnClient.java | 36 +++- .../yarn/client/api/impl/YarnClientImpl.java | 21 ++- .../ApplicationClientProtocolPBClientImpl.java | 37 ++++ .../ApplicationClientProtocolPBServiceImpl.java | 44 +++++ .../pb/GetAttributesToNodesRequestPBImpl.java | 175 ++++++++++++++++++ .../pb/GetAttributesToNodesResponsePBImpl.java | 184 +++++++++++++++++++ .../GetClusterNodeAttributesRequestPBImpl.java | 75 ++++++++ .../GetClusterNodeAttributesResponsePBImpl.java | 156 ++++++++++++++++ .../yarn/nodelabels/NodeAttributesManager.java | 9 +- .../hadoop/yarn/api/TestPBImplRecords.java | 28 +++ .../yarn/server/MockResourceManagerFacade.java | 17 ++ .../server/resourcemanager/ClientRMService.java | 30 +++ .../nodelabels/NodeAttributesManagerImpl.java | 33 ++-- .../resourcemanager/TestClientRMService.java | 126 +++++++++++++ .../DefaultClientRequestInterceptor.java | 17 ++ .../clientrm/FederationClientInterceptor.java | 17 ++ .../router/clientrm/RouterClientRMService.java | 19 ++ .../PassThroughClientRequestInterceptor.java | 17 ++ 29 files changed, 1361 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java index ac4b73b..4f96a6b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java @@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -538,4 +539,16 @@ public class ResourceMgrDelegate extends YarnClient { throws YarnException, IOException { return client.getResourceTypeInfo(); } + + @Override + public Set<NodeAttribute> getClusterAttributes() + throws YarnException, IOException { + return client.getClusterAttributes(); + } + + @Override + public Map<NodeAttribute, Set<String>> getAttributesToNodes( + Set<NodeAttribute> attributes) throws YarnException, IOException { + return client.getAttributesToNodes(attributes); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java index f97d0a4..23a1a85 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java @@ -82,8 +82,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -521,6 +525,19 @@ public class TestClientRedirect { throws YarnException, IOException { return null; } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return null; + } } class HistoryService extends AMService implements HSClientProtocol { http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java index 3c4e4d0..8661a78 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java @@ -27,8 +27,12 @@ import org.apache.hadoop.io.retry.Idempotent; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptRequest; import org.apache.hadoop.yarn.api.protocolrecords.FailApplicationAttemptResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -642,4 +646,37 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol { @Unstable GetAllResourceTypeInfoResponse getResourceTypeInfo( GetAllResourceTypeInfoRequest request) throws YarnException, IOException; + + /** + * <p> + * The interface used by client to get attributes to nodes mappings + * available in ResourceManager. + * </p> + * + * @param request request to get details of attributes to nodes mapping. + * @return Response containing the details of attributes to nodes mappings. + * @throws YarnException if any error happens inside YARN + * @throws IOException incase of other errors + */ + @Public + @Unstable + GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException; + + /** + * <p> + * The interface used by client to get node attributes available in + * ResourceManager. + * </p> + * + * @param request request to get node attributes collection of this cluster. + * @return Response containing node attributes collection. + * @throws YarnException if any error happens inside YARN. + * @throws IOException incase of other errors. + */ + @Public + @Unstable + GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java new file mode 100644 index 0000000..d9531b0 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesRequest.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Set; + +/** + * <p> + * The request from clients to get attribtues to nodes mapping + * in the cluster from the <code>ResourceManager</code>. + * </p> + * + * @see ApplicationClientProtocol#getAttributesToNodes + * (GetAttributesToNodesRequest) + */ +@Public +@Evolving +public abstract class GetAttributesToNodesRequest { + + public static GetAttributesToNodesRequest newInstance() { + return Records.newRecord(GetAttributesToNodesRequest.class); + } + + public static GetAttributesToNodesRequest newInstance( + Set<NodeAttribute> attributes) { + GetAttributesToNodesRequest request = + Records.newRecord(GetAttributesToNodesRequest.class); + request.setNodeAttributes(attributes); + return request; + } + + /** + * Set node attributes for which the mapping is required. + * + * @param attributes Set<NodeAttribute> provided. + */ + @Public + @Unstable + public abstract void setNodeAttributes(Set<NodeAttribute> attributes); + + /** + * Get node attributes for which mapping mapping is required. + * + * @return Set<NodeAttribute> + */ + @Public + @Unstable + public abstract Set<NodeAttribute> getNodeAttributes(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java new file mode 100644 index 0000000..4fdb1f7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetAttributesToNodesResponse.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.Public; +import static org.apache.hadoop.classification.InterfaceStability.Evolving; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Map; +import java.util.Set; + +/** + * <p> + * The response sent by the <code>ResourceManager</code> to a client requesting + * attributes to hostname mapping. + * </p> + * + * @see ApplicationClientProtocol#getAttributesToNodes + * (GetAttributesToNodesRequest) + */ +@Public +@Evolving +public abstract class GetAttributesToNodesResponse { + public static GetAttributesToNodesResponse newInstance( + Map<NodeAttribute, Set<String>> map) { + GetAttributesToNodesResponse response = + Records.newRecord(GetAttributesToNodesResponse.class); + response.setAttributeToNodes(map); + return response; + } + + @Public + @Evolving + public abstract void setAttributeToNodes(Map<NodeAttribute, Set<String>> map); + + /* + * Get attributes to node hostname mapping. + * + * @return Map<NodeAttribute, Set<String>> node attributes to hostname + * mapping. + */ + @Public + @Evolving + public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java new file mode 100644 index 0000000..ca81f9a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesRequest.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.util.Records; + +/** + * <p> + * The request from clients to get node attributes in the cluster from the + * <code>ResourceManager</code>. + * </p> + * + * @see ApplicationClientProtocol#getClusterNodeAttributes + * (GetClusterNodeAttributesRequest) + */ +@Public +@Evolving +public abstract class GetClusterNodeAttributesRequest { + + /** + * Create new instance of GetClusterNodeAttributesRequest. + * + * @return GetClusterNodeAttributesRequest is returned. + */ + public static GetClusterNodeAttributesRequest newInstance() { + return Records.newRecord(GetClusterNodeAttributesRequest.class); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java new file mode 100644 index 0000000..cc3cae4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetClusterNodeAttributesResponse.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.util.Records; + +import java.util.Set; + +/** + * <p> + * The response sent by the <code>ResourceManager</code> to a client requesting + * a node attributes in cluster. + * </p> + * + * @see ApplicationClientProtocol#getClusterNodeAttributes + * (GetClusterNodeAttributesRequest) + */ +@Public +@Evolving +public abstract class GetClusterNodeAttributesResponse { + + /** + * Create instance of GetClusterNodeAttributesResponse. + * + * @param attributes + * @return GetClusterNodeAttributesResponse. + */ + public static GetClusterNodeAttributesResponse newInstance( + Set<NodeAttribute> attributes) { + GetClusterNodeAttributesResponse response = + Records.newRecord(GetClusterNodeAttributesResponse.class); + response.setNodeAttributes(attributes); + return response; + } + + /** + * Set node attributes to the response. + * + * @param attributes Node attributes + */ + @Public + @Unstable + public abstract void setNodeAttributes(Set<NodeAttribute> attributes); + + /** + * Get node attributes of the response. + * + * @return Node attributes + */ + @Public + @Unstable + public abstract Set<NodeAttribute> getNodeAttributes(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 82fede9..2713583 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3391,7 +3391,7 @@ public class YarnConfiguration extends Configuration { public static final String FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS = NODE_ATTRIBUTE_PREFIX + "fs-store.impl.class"; /** - * File system not attribute store directory. + * File system node attribute store directory. */ public static final String FS_NODE_ATTRIBUTE_STORE_ROOT_DIR = NODE_ATTRIBUTE_PREFIX + "fs-store.root-dir"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto index 81adef1..eeb884c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/applicationclient_protocol.proto @@ -64,4 +64,6 @@ service ApplicationClientProtocolService { rpc getResourceProfiles(GetAllResourceProfilesRequestProto) returns (GetAllResourceProfilesResponseProto); rpc getResourceProfile(GetResourceProfileRequestProto) returns (GetResourceProfileResponseProto); rpc getResourceTypeInfo(GetAllResourceTypeInfoRequestProto) returns (GetAllResourceTypeInfoResponseProto); + rpc getClusterNodeAttributes (GetClusterNodeAttributesRequestProto) returns (GetClusterNodeAttributesResponseProto); + rpc getAttributesToNodes (GetAttributesToNodesRequestProto) returns (GetAttributesToNodesResponseProto); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 4bd90b0..8187c44 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -382,6 +382,10 @@ message NodeAttributeProto { optional string attributeValue = 4 [default=""]; } +message AttributeToNodesProto { + required NodeAttributeProto nodeAttribute = 1; + repeated string hostnames = 2; +} enum ContainerTypeProto { APPLICATION_MASTER = 1; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto index 92a65ad..5ed3090 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto @@ -259,6 +259,21 @@ message GetClusterNodeLabelsResponseProto { repeated NodeLabelProto nodeLabels = 2; } +message GetClusterNodeAttributesRequestProto { +} + +message GetClusterNodeAttributesResponseProto { + repeated NodeAttributeProto nodeAttributes = 1; +} + +message GetAttributesToNodesRequestProto { + repeated NodeAttributeProto nodeAttributes = 1; +} + +message GetAttributesToNodesResponseProto { + repeated AttributeToNodesProto attributeToNodes = 1; +} + message UpdateApplicationPriorityRequestProto { required ApplicationIdProto applicationId = 1; required PriorityProto applicationPriority = 2; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java index 26c99e3..ca0b7b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java @@ -33,7 +33,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; -import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationResponse; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest; import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse; @@ -52,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -900,4 +900,38 @@ public abstract class YarnClient extends AbstractService { @Unstable public abstract List<ResourceTypeInfo> getResourceTypeInfo() throws YarnException, IOException; + + /** + * <p> + * The interface used by client to get node attributes in the cluster. + * </p> + * + * @return cluster node attributes collection + * @throws YarnException when there is a failure in + * {@link ApplicationClientProtocol} + * @throws IOException when there is a failure in + * {@link ApplicationClientProtocol} + */ + @Public + @Unstable + public abstract Set<NodeAttribute> getClusterAttributes() + throws YarnException, IOException; + + /** + * <p> + * The interface used by client to get Attributes to nodes mapping + * for specified node attributes in existing cluster. + * </p> + * + * @param attributes Attributes for which Attributes to nodes mapping has to + * be retrieved.If empty or null is set then will return + * all attributes to node mapping in cluster. + * @return Attributes to nodes mappings for specific Attributes. + * @throws YarnException + * @throws IOException + */ + @Public + @Unstable + public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes( + Set<NodeAttribute> attributes) throws YarnException, IOException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index 072e606..eccb7c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -52,8 +51,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; @@ -96,6 +97,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.NodeReport; @@ -104,7 +106,6 @@ import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceTypeInfo; import org.apache.hadoop.yarn.api.records.SignalContainerCommand; import org.apache.hadoop.yarn.api.records.Token; @@ -977,4 +978,20 @@ public class YarnClientImpl extends YarnClient { GetAllResourceTypeInfoRequest.newInstance(); return rmClient.getResourceTypeInfo(request).getResourceTypeInfo(); } + + @Override + public Set<NodeAttribute> getClusterAttributes() + throws YarnException, IOException { + GetClusterNodeAttributesRequest request = + GetClusterNodeAttributesRequest.newInstance(); + return rmClient.getClusterNodeAttributes(request).getNodeAttributes(); + } + + @Override + public Map<NodeAttribute, Set<String>> getAttributesToNodes( + Set<NodeAttribute> attributes) throws YarnException, IOException { + GetAttributesToNodesRequest request = + GetAttributesToNodesRequest.newInstance(attributes); + return rmClient.getAttributesToNodes(request).getAttributesToNodes(); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java index fd5096a..4cf0548 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ApplicationClientProtocolPBClientImpl.java @@ -43,8 +43,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -107,8 +111,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRe import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; @@ -673,4 +681,33 @@ public class ApplicationClientProtocolPBClientImpl implements ApplicationClientP return null; } } + + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + YarnServiceProtos.GetAttributesToNodesRequestProto requestProto = + ((GetAttributesToNodesRequestPBImpl) request).getProto(); + try { + return new GetAttributesToNodesResponsePBImpl( + proxy.getAttributesToNodes(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + YarnServiceProtos.GetClusterNodeAttributesRequestProto requestProto = + ((GetClusterNodeAttributesRequestPBImpl) request).getProto(); + try { + return new GetClusterNodeAttributesResponsePBImpl( + proxy.getClusterNodeAttributes(null, requestProto)); + } catch (ServiceException e) { + RPCUtil.unwrapAndThrowException(e); + return null; + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java index 423287e..8e53f08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ApplicationClientProtocolPBServiceImpl.java @@ -35,7 +35,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRes import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportResponse; @@ -73,8 +76,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRe import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; @@ -184,6 +191,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceTypeInfoResp import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -694,4 +703,39 @@ public class ApplicationClientProtocolPBServiceImpl implements ApplicationClient throw new ServiceException(ie); } } + + @Override + public GetClusterNodeAttributesResponseProto getClusterNodeAttributes( + RpcController controller, + YarnServiceProtos.GetClusterNodeAttributesRequestProto proto) + throws ServiceException { + GetClusterNodeAttributesRequest req = + new GetClusterNodeAttributesRequestPBImpl(proto); + try { + GetClusterNodeAttributesResponse resp = + real.getClusterNodeAttributes(req); + return ((GetClusterNodeAttributesResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } + + @Override + public GetAttributesToNodesResponseProto getAttributesToNodes( + RpcController controller, + YarnServiceProtos.GetAttributesToNodesRequestProto proto) + throws ServiceException { + GetAttributesToNodesRequestPBImpl req = + new GetAttributesToNodesRequestPBImpl(proto); + try { + GetAttributesToNodesResponse resp = real.getAttributesToNodes(req); + return ((GetAttributesToNodesResponsePBImpl) resp).getProto(); + } catch (YarnException ye) { + throw new ServiceException(ye); + } catch (IOException ie) { + throw new ServiceException(ie); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java new file mode 100644 index 0000000..a84fb44 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesRequestPBImpl.java @@ -0,0 +1,175 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.protobuf.TextFormat; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesRequestProto; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; + +/** + * Attributes to nodes mapping request. + */ +@Private +@Unstable +public class GetAttributesToNodesRequestPBImpl + extends GetAttributesToNodesRequest { + + private Set<NodeAttribute> nodeAttributes = null; + + private GetAttributesToNodesRequestProto proto = + GetAttributesToNodesRequestProto.getDefaultInstance(); + private GetAttributesToNodesRequestProto.Builder builder = null; + private boolean viaProto = false; + + public GetAttributesToNodesRequestPBImpl() { + builder = GetAttributesToNodesRequestProto.newBuilder(); + } + + public GetAttributesToNodesRequestPBImpl( + GetAttributesToNodesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetAttributesToNodesRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.nodeAttributes != null) { + addLocalAttributesToProto(); + } + } + + private void addLocalAttributesToProto() { + maybeInitBuilder(); + builder.clearNodeAttributes(); + if (nodeAttributes == null) { + return; + } + Iterable<NodeAttributeProto> iterable = + () -> new Iterator<NodeAttributeProto>() { + private Iterator<NodeAttribute> iter = nodeAttributes.iterator(); + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + + @Override + public NodeAttributeProto next() { + return convertToProtoFormat(iter.next()); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + + } + }; + + builder.addAllNodeAttributes(iterable); + } + + private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) { + return new NodeAttributePBImpl(p); + } + + private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { + return ((NodeAttributePBImpl) t).getProto(); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAttributesToNodesRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void initNodeAttributes() { + if (this.nodeAttributes != null) { + return; + } + YarnServiceProtos.GetAttributesToNodesRequestProtoOrBuilder p = + viaProto ? proto : builder; + List<NodeAttributeProto> nodeAttributesList = p.getNodeAttributesList(); + this.nodeAttributes = new HashSet<>(); + nodeAttributesList + .forEach((v) -> nodeAttributes.add(convertFromProtoFormat(v))); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public void setNodeAttributes(Set<NodeAttribute> attributes) { + maybeInitBuilder(); + if (nodeAttributes == null) { + builder.clearNodeAttributes(); + } + this.nodeAttributes = attributes; + } + + @Override + public Set<NodeAttribute> getNodeAttributes() { + initNodeAttributes(); + return this.nodeAttributes; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java new file mode 100644 index 0000000..ab6204e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetAttributesToNodesResponsePBImpl.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; +import org.apache.hadoop.yarn.proto.YarnProtos.AttributeToNodesProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAttributesToNodesResponseProto; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; + +/** + * Attributes to nodes response. + */ +@Private +@Unstable +public class GetAttributesToNodesResponsePBImpl + extends GetAttributesToNodesResponse { + + private GetAttributesToNodesResponseProto proto = + GetAttributesToNodesResponseProto.getDefaultInstance(); + private GetAttributesToNodesResponseProto.Builder builder = null; + private boolean viaProto = false; + + private Map<NodeAttribute, Set<String>> attributesToNodes; + + public GetAttributesToNodesResponsePBImpl() { + this.builder = GetAttributesToNodesResponseProto.newBuilder(); + } + + public GetAttributesToNodesResponsePBImpl( + GetAttributesToNodesResponseProto proto) { + this.proto = proto; + this.viaProto = true; + } + + private void initAttributesToNodes() { + if (this.attributesToNodes != null) { + return; + } + YarnServiceProtos.GetAttributesToNodesResponseProtoOrBuilder p = + viaProto ? proto : builder; + List<AttributeToNodesProto> list = p.getAttributeToNodesList(); + this.attributesToNodes = new HashMap<>(); + + for (AttributeToNodesProto c : list) { + Set<String> setNodes = new HashSet<>(c.getHostnamesList()); + if (!setNodes.isEmpty()) { + this.attributesToNodes + .put(convertFromProtoFormat(c.getNodeAttribute()), setNodes); + } + } + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetAttributesToNodesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void addAttributesToNodesToProto() { + maybeInitBuilder(); + builder.clearAttributeToNodes(); + if (attributesToNodes == null) { + return; + } + Iterable<AttributeToNodesProto> iterable = + () -> new Iterator<AttributeToNodesProto>() { + + private Iterator<Map.Entry<NodeAttribute, Set<String>>> iter = + attributesToNodes.entrySet().iterator(); + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public AttributeToNodesProto next() { + Map.Entry<NodeAttribute, Set<String>> now = iter.next(); + Set<String> hostNames = new HashSet<>(); + for (String host : now.getValue()) { + hostNames.add(host); + } + return AttributeToNodesProto.newBuilder() + .setNodeAttribute(convertToProtoFormat(now.getKey())) + .addAllHostnames(hostNames).build(); + } + + @Override + public boolean hasNext() { + return iter.hasNext(); + } + }; + builder.addAllAttributeToNodes(iterable); + } + + private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) { + return new NodeAttributePBImpl(p); + } + + private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { + return ((NodeAttributePBImpl) t).getProto(); + } + + private void mergeLocalToBuilder() { + if (this.attributesToNodes != null) { + addAttributesToNodesToProto(); + } + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + public GetAttributesToNodesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 0; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public void setAttributeToNodes(Map<NodeAttribute, Set<String>> map) { + initAttributesToNodes(); + attributesToNodes.clear(); + attributesToNodes.putAll(map); + } + + @Override + public Map<NodeAttribute, Set<String>> getAttributesToNodes() { + initAttributesToNodes(); + return this.attributesToNodes; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java new file mode 100644 index 0000000..bf5ab40 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesRequestPBImpl.java @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import com.google.protobuf.TextFormat; +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesRequestProto; + +/** + * Request to get cluster node attributes. + */ +@Private +@Unstable +public class GetClusterNodeAttributesRequestPBImpl + extends GetClusterNodeAttributesRequest { + + private GetClusterNodeAttributesRequestProto proto = + GetClusterNodeAttributesRequestProto.getDefaultInstance(); + private GetClusterNodeAttributesRequestProto.Builder builder = null; + private boolean viaProto = false; + + public GetClusterNodeAttributesRequestPBImpl() { + builder = GetClusterNodeAttributesRequestProto.newBuilder(); + } + + public GetClusterNodeAttributesRequestPBImpl( + GetClusterNodeAttributesRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public GetClusterNodeAttributesRequestProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java new file mode 100644 index 0000000..385155f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetClusterNodeAttributesResponsePBImpl.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.api.protocolrecords.impl.pb; + +import static org.apache.hadoop.classification.InterfaceAudience.*; +import static org.apache.hadoop.classification.InterfaceStability.*; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl; +import org.apache.hadoop.yarn.proto.YarnServiceProtos; +import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetClusterNodeAttributesResponseProto; +import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Cluster node attributes response. + */ +@Private +@Unstable +public class GetClusterNodeAttributesResponsePBImpl + extends GetClusterNodeAttributesResponse { + + private GetClusterNodeAttributesResponseProto proto = + GetClusterNodeAttributesResponseProto.getDefaultInstance(); + private GetClusterNodeAttributesResponseProto.Builder builder = null; + private Set<NodeAttribute> updatedNodeAttributes; + private boolean viaProto = false; + + public GetClusterNodeAttributesResponsePBImpl() { + builder = GetClusterNodeAttributesResponseProto.newBuilder(); + } + + public GetClusterNodeAttributesResponsePBImpl( + GetClusterNodeAttributesResponseProto proto) { + this.proto = proto; + viaProto = true; + } + + public synchronized GetClusterNodeAttributesResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.updatedNodeAttributes != null) { + addNodeAttributesToProto(); + } + } + + private void addNodeAttributesToProto() { + maybeInitBuilder(); + builder.clearNodeAttributes(); + List<NodeAttributeProto> protoList = new ArrayList<>(); + for (NodeAttribute r : this.updatedNodeAttributes) { + protoList.add(convertToProtoFormat(r)); + } + builder.addAllNodeAttributes(protoList); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public int hashCode() { + assert false : "hashCode not designed"; + return 0; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = GetClusterNodeAttributesResponseProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public synchronized void setNodeAttributes(Set<NodeAttribute> attributes) { + maybeInitBuilder(); + this.updatedNodeAttributes = new HashSet<>(); + if (attributes == null) { + builder.clearNodeAttributes(); + return; + } + this.updatedNodeAttributes.addAll(attributes); + } + + @Override + public synchronized Set<NodeAttribute> getNodeAttributes() { + if (this.updatedNodeAttributes != null) { + return this.updatedNodeAttributes; + } + initLocalNodeAttributes(); + return this.updatedNodeAttributes; + } + + private void initLocalNodeAttributes() { + YarnServiceProtos.GetClusterNodeAttributesResponseProtoOrBuilder p = + viaProto ? proto : builder; + List<NodeAttributeProto> attributesProtoList = p.getNodeAttributesList(); + this.updatedNodeAttributes = new HashSet<>(); + for (NodeAttributeProto r : attributesProtoList) { + this.updatedNodeAttributes.add(convertFromProtoFormat(r)); + } + } + + private NodeAttribute convertFromProtoFormat(NodeAttributeProto p) { + return new NodeAttributePBImpl(p); + } + + private NodeAttributeProto convertToProtoFormat(NodeAttribute t) { + return ((NodeAttributePBImpl) t).getProto(); + } + + @Override + public String toString() { + return getProto().toString(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java index ec7d30d..3816051 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java @@ -88,12 +88,13 @@ public abstract class NodeAttributesManager extends AbstractService { /** * Given a attribute set, return what all Nodes have attribute mapped to it. + * If the attributes set is null or empty, all attributes mapping are + * returned. * - * @return a Map, of attribute to set of hostnames + * @return a Map of attributes to set of hostnames. */ - //TODO need to handle as part of REST patch. -/* public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes( - Set<NodeAttribute> attributes);*/ + public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes( + Set<NodeAttribute> attributes); /** * NodeAttribute to AttributeValue Map. http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java index f6f0d74..f100869 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java @@ -53,8 +53,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportRe import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationReportResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetApplicationsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetAttributesToNodesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterMetricsResponsePBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesRequestPBImpl; +import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeAttributesResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsRequestPBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodeLabelsResponsePBImpl; import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetClusterNodesRequestPBImpl; @@ -1259,4 +1263,28 @@ public class TestPBImplRecords extends BasePBImplRecordsTest { validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class, NodesToAttributesMappingRequestProto.class); } + + @Test + public void testGetAttributesToNodesRequestPBImpl() throws Exception { + validatePBImplRecord(GetAttributesToNodesRequestPBImpl.class, + YarnServiceProtos.GetAttributesToNodesRequestProto.class); + } + + @Test + public void testGetAttributesToNodesResponsePBImpl() throws Exception { + validatePBImplRecord(GetAttributesToNodesResponsePBImpl.class, + YarnServiceProtos.GetAttributesToNodesResponseProto.class); + } + + @Test + public void testGetClusterNodeAttributesRequestPBImpl() throws Exception { + validatePBImplRecord(GetClusterNodeAttributesRequestPBImpl.class, + YarnServiceProtos.GetClusterNodeAttributesRequestProto.class); + } + + @Test + public void testGetClusterNodeAttributesResponsePBImpl() throws Exception { + validatePBImplRecord(GetClusterNodeAttributesResponsePBImpl.class, + YarnServiceProtos.GetClusterNodeAttributesResponseProto.class); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java index 38bd2a9..ee2a91d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java @@ -53,8 +53,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -902,6 +906,19 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol, } @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + return null; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + return null; + } + + @Override public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request) throws YarnException, IOException { return null; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 55a3f0b..2e36fe9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -66,8 +66,12 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; @@ -127,6 +131,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.NodeAttribute; import org.apache.hadoop.yarn.api.records.NodeReport; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Priority; @@ -148,6 +153,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; @@ -1834,6 +1840,30 @@ public class ClientRMService extends AbstractService implements return response; } + @Override + public GetAttributesToNodesResponse getAttributesToNodes( + GetAttributesToNodesRequest request) throws YarnException, IOException { + NodeAttributesManager attributesManager = + rmContext.getNodeAttributesManager(); + GetAttributesToNodesResponse response = GetAttributesToNodesResponse + .newInstance(attributesManager + .getAttributesToNodes(request.getNodeAttributes())); + return response; + } + + @Override + public GetClusterNodeAttributesResponse getClusterNodeAttributes( + GetClusterNodeAttributesRequest request) + throws YarnException, IOException { + NodeAttributesManager attributesManager = + rmContext.getNodeAttributesManager(); + Set<NodeAttribute> attributes = + attributesManager.getClusterNodeAttributes(null); + GetClusterNodeAttributesResponse response = + GetClusterNodeAttributesResponse.newInstance(attributes); + return response; + } + @VisibleForTesting public void setDisplayPerUserApps(boolean displayPerUserApps) { this.displayPerUserApps = displayPerUserApps; http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java index b4686e6..510cbaf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java @@ -367,19 +367,26 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager { return attributes; } - // TODO need to handle as part of REST patch. - /* - * @Override public Map<NodeAttribute, Set<String>> getAttributesToNodes( - * Set<NodeAttribute> attributes) { try { readLock.lock(); boolean - * fetchAllAttributes = (attributes == null || attributes.isEmpty()); - * Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>(); for - * (Entry<NodeAttribute, RMAttributeNodeLabel> attributeEntry : - * attributeCollections .entrySet()) { if (fetchAllAttributes || - * attributes.contains(attributeEntry.getKey())) { - * attributesToNodes.put(attributeEntry.getKey(), - * attributeEntry.getValue().getAssociatedNodeIds()); } } return - * attributesToNodes; } finally { readLock.unlock(); } } - */ + @Override + public Map<NodeAttribute, Set<String>> getAttributesToNodes( + Set<NodeAttribute> attributes) { + try { + readLock.lock(); + boolean fetchAllAttributes = (attributes == null || attributes.isEmpty()); + Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>(); + for (Entry<NodeAttribute, RMNodeAttribute> attributeEntry : + clusterAttributes.entrySet()) { + if (fetchAllAttributes || attributes + .contains(attributeEntry.getKey())) { + attributesToNodes.put(attributeEntry.getKey(), + attributeEntry.getValue().getAssociatedNodeIds()); + } + } + return attributesToNodes; + } finally { + readLock.unlock(); + } + } public Resource getResourceByAttribute(NodeAttribute attribute) { try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/efcca85f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java index 1c50dd3..a811ef2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java @@ -18,6 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse; +import org.apache.hadoop.yarn.api.records.NodeAttribute; +import org.apache.hadoop.yarn.api.records.NodeAttributeType; +import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; @@ -1992,6 +1999,125 @@ public class TestClientRMService { } @Test(timeout = 120000) + public void testGetClusterNodeAttributes() throws IOException, YarnException { + MockRM rm = new MockRM() { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, + this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + } + }; + rm.start(); + + NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager(); + NodeId host1 = NodeId.newInstance("host1", 0); + NodeId host2 = NodeId.newInstance("host2", 0); + NodeAttribute gpu = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvida"); + NodeAttribute os = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttributeType.STRING, "windows64"); + NodeAttribute docker = NodeAttribute + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + Map<String, Set<NodeAttribute>> nodes = new HashMap<>(); + nodes.put(host1.getHost(), ImmutableSet.of(gpu, os)); + nodes.put(host2.getHost(), ImmutableSet.of(docker)); + mgr.addNodeAttributes(nodes); + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + GetClusterNodeAttributesRequest request = + GetClusterNodeAttributesRequest.newInstance(); + GetClusterNodeAttributesResponse response = + client.getClusterNodeAttributes(request); + Set<NodeAttribute> attributes = response.getNodeAttributes(); + Assert.assertEquals("Size not correct", 3, attributes.size()); + Assert.assertTrue(attributes.contains(gpu)); + Assert.assertTrue(attributes.contains(os)); + Assert.assertTrue(attributes.contains(docker)); + rpc.stopProxy(client, conf); + rm.close(); + } + + @Test(timeout = 120000) + public void testGetAttributesToNodes() throws IOException, YarnException { + MockRM rm = new MockRM() { + protected ClientRMService createClientRMService() { + return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, + this.applicationACLsManager, this.queueACLsManager, + this.getRMContext().getRMDelegationTokenSecretManager()); + } + }; + rm.start(); + + NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager(); + String node1 = "host1"; + String node2 = "host2"; + NodeAttribute gpu = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU", + NodeAttributeType.STRING, "nvida"); + NodeAttribute os = NodeAttribute + .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS", + NodeAttributeType.STRING, "windows64"); + NodeAttribute docker = NodeAttribute + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER", + NodeAttributeType.STRING, "docker0"); + NodeAttribute dist = NodeAttribute + .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION", + NodeAttributeType.STRING, "3_0_2"); + Map<String, Set<NodeAttribute>> nodes = new HashMap<>(); + nodes.put(node1, ImmutableSet.of(gpu, os, dist)); + nodes.put(node2, ImmutableSet.of(docker, dist)); + mgr.addNodeAttributes(nodes); + // Create a client. + Configuration conf = new Configuration(); + YarnRPC rpc = YarnRPC.create(conf); + InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress(); + LOG.info("Connecting to ResourceManager at " + rmAddress); + ApplicationClientProtocol client = (ApplicationClientProtocol) rpc + .getProxy(ApplicationClientProtocol.class, rmAddress, conf); + + GetAttributesToNodesRequest request = + GetAttributesToNodesRequest.newInstance(); + GetAttributesToNodesResponse response = + client.getAttributesToNodes(request); + Map<NodeAttribute, Set<String>> attrs = response.getAttributesToNodes(); + Assert.assertEquals(response.getAttributesToNodes().size(), 4); + Assert.assertEquals(attrs.get(dist).size(), 2); + Assert.assertEquals(attrs.get(os).size(), 1); + Assert.assertEquals(attrs.get(gpu).size(), 1); + Assert.assertTrue(attrs.get(dist).contains(node1)); + Assert.assertTrue(attrs.get(dist).contains(node2)); + Assert.assertTrue(attrs.get(docker).contains(node2)); + + GetAttributesToNodesRequest request2 = + GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker)); + GetAttributesToNodesResponse response2 = + client.getAttributesToNodes(request2); + Map<NodeAttribute, Set<String>> attrs2 = response2.getAttributesToNodes(); + Assert.assertEquals(response2.getAttributesToNodes().size(), 1); + Assert.assertTrue(attrs.get(docker).contains(node2)); + + GetAttributesToNodesRequest request3 = + GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os)); + GetAttributesToNodesResponse response3 = + client.getAttributesToNodes(request3); + Map<NodeAttribute, Set<String>> attrs3 = response3.getAttributesToNodes(); + Assert.assertEquals(response3.getAttributesToNodes().size(), 2); + Assert.assertTrue(attrs.get(os).contains(node1)); + Assert.assertTrue(attrs.get(docker).contains(node2)); + rpc.stopProxy(client, conf); + rm.close(); + } + + @Test(timeout = 120000) public void testUpdatePriorityAndKillAppWithZeroClusterResource() throws Exception { int maxPriority = 10; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org