[
https://issues.apache.org/jira/browse/YARN-11577?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17786785#comment-17786785
]
ASF GitHub Bot commented on YARN-11577:
---------------------------------------
slfan1989 commented on code in PR #6190:
URL: https://github.com/apache/hadoop/pull/6190#discussion_r1395847402
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/capacity/TestYarnFederationWithCapacityScheduler.java:
##########
@@ -73,4 +147,469 @@ public void testGetClusterInfo() throws
InterruptedException, IOException {
assertTrue(subClusters.contains(clusterInfo.getSubClusterId()));
}
}
+
+ @Test
+ public void testInfo() throws InterruptedException, IOException {
+ FederationClusterInfo federationClusterInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
RM_WEB_SERVICE_PATH + INFO,
+ FederationClusterInfo.class, null, null);
+ List<ClusterInfo> clusterInfos = federationClusterInfo.getList();
+ assertNotNull(clusterInfos);
+ assertEquals(2, clusterInfos.size());
+ for (ClusterInfo clusterInfo : clusterInfos) {
+ assertNotNull(clusterInfo);
+ assertTrue(subClusters.contains(clusterInfo.getSubClusterId()));
+ }
+ }
+
+ @Test
+ public void testClusterUserInfo() throws Exception {
+ FederationClusterUserInfo federationClusterUserInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + CLUSTER_USER_INFO,
+ FederationClusterUserInfo.class, null, null);
+ List<ClusterUserInfo> clusterUserInfos =
federationClusterUserInfo.getList();
+ assertNotNull(clusterUserInfos);
+ assertEquals(2, clusterUserInfos.size());
+ for (ClusterUserInfo clusterUserInfo : clusterUserInfos) {
+ assertNotNull(clusterUserInfo);
+ assertTrue(subClusters.contains(clusterUserInfo.getSubClusterId()));
+ }
+ }
+
+ @Test
+ public void testMetricsInfo() throws Exception {
+ // It takes time to start the sub-cluster.
+ // We need to wait for the sub-cluster to be completely started,
+ // so we need to set the waiting time.
+ // The resources of the two sub-clusters we registered are 24C and 12G,
+ // so the resources that the Router should collect are 48C and 24G.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ClusterMetricsInfo clusterMetricsInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + METRICS, ClusterMetricsInfo.class, null,
null);
+ assertNotNull(clusterMetricsInfo);
+ return (48 == clusterMetricsInfo.getTotalVirtualCores() &&
+ 24576 == clusterMetricsInfo.getTotalMB());
+ } catch (Exception e) {
+ return false;
+ }
+ }, 5000, 50 * 5000);
+ }
+
+ @Test
+ public void testSchedulerInfo() throws Exception {
+ FederationSchedulerTypeInfo schedulerTypeInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + SCHEDULER, FederationSchedulerTypeInfo.class,
null, null);
+ assertNotNull(schedulerTypeInfo);
+ List<SchedulerTypeInfo> schedulerTypeInfos = schedulerTypeInfo.getList();
+ assertNotNull(schedulerTypeInfos);
+ assertEquals(2, schedulerTypeInfos.size());
+ for (SchedulerTypeInfo schedulerTypeInfoItem : schedulerTypeInfos) {
+ assertNotNull(schedulerTypeInfoItem);
+
assertTrue(subClusters.contains(schedulerTypeInfoItem.getSubClusterId()));
+ CapacitySchedulerInfo schedulerInfo =
+ (CapacitySchedulerInfo) schedulerTypeInfoItem.getSchedulerInfo();
+ assertNotNull(schedulerInfo);
+ assertEquals(3, schedulerInfo.getQueues().getQueueInfoList().size());
+ }
+ }
+
+ @Test
+ public void testNodesEmpty() throws Exception {
+ // We are in 2 sub-clusters, each with 3 nodes, so our Router should
correspond to 6 nodes.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ NodesInfo nodesInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + NODES, NodesInfo.class, null, null);
+ assertNotNull(nodesInfo);
+ ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
+ assertNotNull(nodes);
+ return (6 == nodes.size());
+ } catch (Exception e) {
+ return false;
+ }
+ }, 5000, 50 * 5000);
+ }
+
+ @Test
+ public void testNodesLost() throws Exception {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ NodesInfo nodesInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + NODES, NodesInfo.class, STATES, "LOST");
+ assertNotNull(nodesInfo);
+ ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
+ assertNotNull(nodes);
+ return (0 == nodes.size());
Review Comment:
Thanks for your suggestion! I will improve it.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/subcluster/fair/TestYarnFederationWithFairScheduler.java:
##########
@@ -72,4 +152,488 @@ public void testGetClusterInfo() throws
InterruptedException, IOException {
assertTrue(subClusters.contains(clusterInfo.getSubClusterId()));
}
}
+
+ @Test
+ public void testInfo() throws InterruptedException, IOException {
+ FederationClusterInfo federationClusterInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
RM_WEB_SERVICE_PATH + INFO,
+ FederationClusterInfo.class, null, null);
+ List<ClusterInfo> clusterInfos = federationClusterInfo.getList();
+ assertNotNull(clusterInfos);
+ assertEquals(2, clusterInfos.size());
+ for (ClusterInfo clusterInfo : clusterInfos) {
+ assertNotNull(clusterInfo);
+ assertTrue(subClusters.contains(clusterInfo.getSubClusterId()));
+ }
+ }
+
+ @Test
+ public void testClusterUserInfo() throws Exception {
+ FederationClusterUserInfo federationClusterUserInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + CLUSTER_USER_INFO,
+ FederationClusterUserInfo.class, null, null);
+ List<ClusterUserInfo> clusterUserInfos =
federationClusterUserInfo.getList();
+ assertNotNull(clusterUserInfos);
+ assertEquals(2, clusterUserInfos.size());
+ for (ClusterUserInfo clusterUserInfo : clusterUserInfos) {
+ assertNotNull(clusterUserInfo);
+ assertTrue(subClusters.contains(clusterUserInfo.getSubClusterId()));
+ }
+ }
+
+ @Test
+ public void testMetricsInfo() throws Exception {
+ // It takes time to start the sub-cluster.
+ // We need to wait for the sub-cluster to be completely started,
+ // so we need to set the waiting time.
+ // The resources of the two sub-clusters we registered are 24C and 12G,
+ // so the resources that the Router should collect are 48C and 24G.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ ClusterMetricsInfo clusterMetricsInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + METRICS, ClusterMetricsInfo.class, null,
null);
+ assertNotNull(clusterMetricsInfo);
+ return (48 == clusterMetricsInfo.getTotalVirtualCores() &&
+ 24576 == clusterMetricsInfo.getTotalMB());
+ } catch (Exception e) {
+ return false;
+ }
+ }, 5000, 50 * 5000);
+ }
+
+ @Test
+ public void testSchedulerInfo() throws Exception {
+ FederationSchedulerTypeInfo schedulerTypeInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + SCHEDULER, FederationSchedulerTypeInfo.class,
null, null);
+ assertNotNull(schedulerTypeInfo);
+ List<SchedulerTypeInfo> schedulerTypeInfos = schedulerTypeInfo.getList();
+ assertNotNull(schedulerTypeInfos);
+ assertEquals(2, schedulerTypeInfos.size());
+ for (SchedulerTypeInfo schedulerTypeInfoItem : schedulerTypeInfos) {
+ assertNotNull(schedulerTypeInfoItem);
+
assertTrue(subClusters.contains(schedulerTypeInfoItem.getSubClusterId()));
+ FairSchedulerQueueInfo rootQueueInfo =
+ ((FairSchedulerInfo)
schedulerTypeInfoItem.getSchedulerInfo()).getRootQueueInfo();
+ assertNotNull(rootQueueInfo);
+ assertEquals("fair", rootQueueInfo.getSchedulingPolicy());
+ }
+ }
+
+ @Test
+ public void testNodesEmpty() throws Exception {
+ // We are in 2 sub-clusters, each with 3 nodes, so our Router should
correspond to 6 nodes.
+ GenericTestUtils.waitFor(() -> {
+ try {
+ NodesInfo nodesInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + NODES, NodesInfo.class, null, null);
+ assertNotNull(nodesInfo);
+ ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
+ assertNotNull(nodes);
+ return (6 == nodes.size());
+ } catch (Exception e) {
+ return false;
+ }
+ }, 5000, 50 * 5000);
+ }
+
+ @Test
+ public void testNodesLost() throws Exception {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ NodesInfo nodesInfo =
+ TestFederationSubCluster.performGetCalls(ROUTER_WEB_ADDRESS,
+ RM_WEB_SERVICE_PATH + NODES, NodesInfo.class, STATES, "LOST");
+ assertNotNull(nodesInfo);
+ ArrayList<NodeInfo> nodes = nodesInfo.getNodes();
+ assertNotNull(nodes);
+ return (0 == nodes.size());
Review Comment:
I will improve it.
> Improve FederationInterceptorREST Method Result
> -----------------------------------------------
>
> Key: YARN-11577
> URL: https://issues.apache.org/jira/browse/YARN-11577
> Project: Hadoop YARN
> Issue Type: Improvement
> Components: federation
> Affects Versions: 3.4.0
> Reporter: Shilun Fan
> Assignee: Shilun Fan
> Priority: Major
> Labels: pull-request-available
>
> In the FederationInterceptorREST, some methods currently return null or throw
> exceptions. We will ensure that the returned results are similar to those
> returned by RM, handling cases of null values or exceptions gracefully.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
