This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fe3984aa010 YARN-11580. YARN Router Web supports displaying
information for Non-Federation. (#6127) Contributed by Shilun Fan.
fe3984aa010 is described below
commit fe3984aa010b7a470bf3dcb00029369435382d1b
Author: slfan1989 <55643692+slfan1...@users.noreply.github.com>
AuthorDate: Tue Oct 3 18:21:52 2023 +0800
YARN-11580. YARN Router Web supports displaying information for
Non-Federation. (#6127) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <inigo...@apache.org>
Signed-off-by: Shilun Fan <slfan1...@apache.org>
---
.../federation/store/records/SubClusterInfo.java | 7 +
.../yarn/server/router/RouterServerUtil.java | 1 +
.../yarn/server/router/webapp/AppsBlock.java | 10 +-
.../yarn/server/router/webapp/FederationBlock.java | 203 +++++++++++++--------
.../server/router/webapp/MetricsOverviewTable.java | 158 ++++++++++++----
.../hadoop/yarn/server/router/webapp/NavBlock.java | 5 +-
.../yarn/server/router/webapp/NodeLabelsBlock.java | 32 +++-
.../yarn/server/router/webapp/NodesBlock.java | 11 +-
.../yarn/server/router/webapp/RouterBlock.java | 133 ++++++++++++--
.../router/webapp/dao/RouterSchedulerMetrics.java | 13 +-
.../server/router/webapp/TestFederationWebApp.java | 2 +
.../src/test/resources/yarn-site.xml | 4 +
12 files changed, 439 insertions(+), 140 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
index 40b87c7eb09..f5a8f04f167 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/SubClusterInfo.java
@@ -76,6 +76,13 @@ public abstract class SubClusterInfo {
return subClusterInfo;
}
+ public static SubClusterInfo newInstance(SubClusterId subClusterId,
+ String rmWebServiceAddress, SubClusterState state, long lastStartTime,
long lastHeartBeat,
+ String capability) {
+ return newInstance(subClusterId, null, null, null,
+ rmWebServiceAddress, lastHeartBeat, state, lastStartTime, capability);
+ }
+
/**
* Get the {@link SubClusterId} representing the unique identifier of the
* subcluster.
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
index 94dbcdce1ae..096d8dac0a7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterServerUtil.java
@@ -661,6 +661,7 @@ public final class RouterServerUtil {
* - if its size is within limits.
*
* @param appContext the app context to check.
+ * @param conf Configuration.
* @throws IOException if an IO error occurred.
* @throws YarnException yarn exception.
*/
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
index 49ad710bb11..4fa07b70fa7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/AppsBlock.java
@@ -94,11 +94,13 @@ public class AppsBlock extends RouterBlock {
}
private AppsInfo getYarnFederationAppsInfo(boolean isEnabled) {
+ String webAddress = null;
if (isEnabled) {
- String webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
- return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
+ webAddress = WebAppUtils.getRouterWebAppURLWithScheme(this.conf);
+ } else {
+ webAddress = WebAppUtils.getRMWebAppURLWithScheme(this.conf);
}
- return null;
+ return getSubClusterAppsInfoByWebAddress(webAddress, StringUtils.EMPTY);
}
private AppsInfo getSubClusterAppsInfo(String subCluster, String states) {
@@ -110,7 +112,7 @@ public class AppsBlock extends RouterBlock {
if (subClusterInfo != null) {
// Prepare webAddress
String webAddress = subClusterInfo.getRMWebServiceAddress();
- String herfWebAppAddress = "";
+ String herfWebAppAddress;
if (webAddress != null && !webAddress.isEmpty()) {
herfWebAppAddress = WebAppUtils.getHttpSchemePrefix(conf) +
webAddress;
return getSubClusterAppsInfoByWebAddress(herfWebAppAddress, states);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
index 44f8d7407bf..7876d5d6f1f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationBlock.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Date;
import com.google.gson.Gson;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
@@ -67,7 +68,7 @@ class FederationBlock extends RouterBlock {
* @param capability metric json obtained from RM.
* @return ClusterMetricsInfo Object
*/
- private ClusterMetricsInfo getClusterMetricsInfo(String capability) {
+ protected ClusterMetricsInfo getClusterMetricsInfo(String capability) {
try {
if (capability != null && !capability.isEmpty()) {
JSONJAXBContext jc = new JSONJAXBContext(
@@ -125,79 +126,10 @@ class FederationBlock extends RouterBlock {
.__().__().tbody();
try {
-
- // Sort the SubClusters
- List<SubClusterInfo> subclusters = getSubClusterInfoList();
-
- for (SubClusterInfo subcluster : subclusters) {
-
- Map<String, String> subclusterMap = new HashMap<>();
-
- // Prepare subCluster
- SubClusterId subClusterId = subcluster.getSubClusterId();
- String subClusterIdText = subClusterId.getId();
-
- // Prepare WebAppAddress
- String webAppAddress = subcluster.getRMWebServiceAddress();
- String herfWebAppAddress = "";
- if (webAppAddress != null && !webAppAddress.isEmpty()) {
- herfWebAppAddress =
- WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) +
webAppAddress;
- }
-
- // Prepare Capability
- String capability = subcluster.getCapability();
- ClusterMetricsInfo subClusterInfo = getClusterMetricsInfo(capability);
-
- // Prepare LastStartTime & LastHeartBeat
- Date lastStartTime = new Date(subcluster.getLastStartTime());
- Date lastHeartBeat = new Date(subcluster.getLastHeartBeat());
-
- // Prepare Resource
- long totalMB = subClusterInfo.getTotalMB();
- String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB);
- long totalVirtualCores = subClusterInfo.getTotalVirtualCores();
- String resources = String.format("<memory:%s, vCores:%s>",
totalMBDesc, totalVirtualCores);
-
- // Prepare Node
- long totalNodes = subClusterInfo.getTotalNodes();
- long activeNodes = subClusterInfo.getActiveNodes();
- String nodes = String.format("<totalNodes:%s, activeNodes:%s>",
totalNodes, activeNodes);
-
- // Prepare HTML Table
- String stateStyle = "color:#dc3545;font-weight:bolder";
- SubClusterState state = subcluster.getState();
- if (SubClusterState.SC_RUNNING == state) {
- stateStyle = "color:#28a745;font-weight:bolder";
- }
-
- tbody.tr().$id(subClusterIdText)
- .td().$class("details-control").a(herfWebAppAddress,
subClusterIdText).__()
- .td().$style(stateStyle).__(state.name()).__()
- .td().__(lastStartTime).__()
- .td().__(lastHeartBeat).__()
- .td(resources)
- .td(nodes)
- .__();
-
- // Formatted memory information
- long allocatedMB = subClusterInfo.getAllocatedMB();
- String allocatedMBDesc = StringUtils.byteDesc(allocatedMB *
BYTES_IN_MB);
- long availableMB = subClusterInfo.getAvailableMB();
- String availableMBDesc = StringUtils.byteDesc(availableMB *
BYTES_IN_MB);
- long pendingMB = subClusterInfo.getPendingMB();
- String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB);
- long reservedMB = subClusterInfo.getReservedMB();
- String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB);
-
- subclusterMap.put("totalmemory", totalMBDesc);
- subclusterMap.put("allocatedmemory", allocatedMBDesc);
- subclusterMap.put("availablememory", availableMBDesc);
- subclusterMap.put("pendingmemory", pendingMBDesc);
- subclusterMap.put("reservedmemory", reservedMBDesc);
- subclusterMap.put("subcluster", subClusterId.getId());
- subclusterMap.put("capability", capability);
- lists.add(subclusterMap);
+ if (isEnabled) {
+ initSubClusterPage(tbody, lists);
+ } else {
+ initLocalClusterPage(tbody, lists);
}
} catch (Exception e) {
LOG.error("Cannot render Router Federation.", e);
@@ -210,4 +142,127 @@ class FederationBlock extends RouterBlock {
tbody.__().__().div().p().$style("color:red")
.__("*The application counts are local per subcluster").__().__();
}
+
+ /**
+ * Initialize the Federation page of the local-cluster.
+ *
+ * @param tbody HTML tbody.
+ * @param lists subCluster page data list.
+ */
+ private void initLocalClusterPage(TBODY<TABLE<Hamlet>> tbody,
List<Map<String, String>> lists) {
+ Configuration config = this.router.getConfig();
+ SubClusterInfo localCluster = getSubClusterInfoByLocalCluster(config);
+ if (localCluster != null) {
+ try {
+ initSubClusterPageItem(tbody, localCluster, lists);
+ } catch (Exception e) {
+ LOG.error("init LocalCluster = {} page data error.", localCluster, e);
+ }
+ }
+ }
+
+ /**
+ * Initialize the Federation page of the sub-cluster.
+ *
+ * @param tbody HTML tbody.
+ * @param lists subCluster page data list.
+ */
+ private void initSubClusterPage(TBODY<TABLE<Hamlet>> tbody, List<Map<String,
String>> lists) {
+ // Sort the SubClusters
+ List<SubClusterInfo> subClusters = getSubClusterInfoList();
+
+ // Iterate through the sub-clusters and display data for each sub-cluster.
+ // If a sub-cluster cannot display data, skip it.
+ for (SubClusterInfo subCluster : subClusters) {
+ try {
+ initSubClusterPageItem(tbody, subCluster, lists);
+ } catch (Exception e) {
+ LOG.error("init subCluster = {} page data error.", subCluster, e);
+ }
+ }
+ }
+
+ /**
+ * We will initialize the specific SubCluster's data within this method.
+ *
+ * @param tbody HTML TBody.
+ * @param subClusterInfo Sub-cluster information.
+ * @param lists Used to record data that needs to be displayed in JS.
+ */
+ private void initSubClusterPageItem(TBODY<TABLE<Hamlet>> tbody,
+ SubClusterInfo subClusterInfo, List<Map<String, String>> lists) {
+
+ Map<String, String> subClusterMap = new HashMap<>();
+
+ // Prepare subCluster
+ SubClusterId subClusterId = subClusterInfo.getSubClusterId();
+ String subClusterIdText = subClusterId.getId();
+
+ // Prepare WebAppAddress
+ String webAppAddress = subClusterInfo.getRMWebServiceAddress();
+ String herfWebAppAddress = "";
+ if (webAppAddress != null && !webAppAddress.isEmpty()) {
+ herfWebAppAddress =
+ WebAppUtils.getHttpSchemePrefix(this.router.getConfig()) +
webAppAddress;
+ }
+
+ // Prepare Capability
+ String capability = subClusterInfo.getCapability();
+ ClusterMetricsInfo subClusterMetricsInfo =
getClusterMetricsInfo(capability);
+
+ if (subClusterMetricsInfo == null) {
+ return;
+ }
+
+ // Prepare LastStartTime & LastHeartBeat
+ Date lastStartTime = new Date(subClusterInfo.getLastStartTime());
+ Date lastHeartBeat = new Date(subClusterInfo.getLastHeartBeat());
+
+ // Prepare Resource
+ long totalMB = subClusterMetricsInfo.getTotalMB();
+ String totalMBDesc = StringUtils.byteDesc(totalMB * BYTES_IN_MB);
+ long totalVirtualCores = subClusterMetricsInfo.getTotalVirtualCores();
+ String resources = String.format("<memory:%s, vCores:%s>", totalMBDesc,
totalVirtualCores);
+
+ // Prepare Node
+ long totalNodes = subClusterMetricsInfo.getTotalNodes();
+ long activeNodes = subClusterMetricsInfo.getActiveNodes();
+ String nodes = String.format("<totalNodes:%s, activeNodes:%s>",
totalNodes, activeNodes);
+
+ // Prepare HTML Table
+ String stateStyle = "color:#dc3545;font-weight:bolder";
+ SubClusterState state = subClusterInfo.getState();
+ if (SubClusterState.SC_RUNNING == state) {
+ stateStyle = "color:#28a745;font-weight:bolder";
+ }
+
+ tbody.tr().$id(subClusterIdText)
+ .td().$class("details-control").a(herfWebAppAddress,
subClusterIdText).__()
+ .td().$style(stateStyle).__(state.name()).__()
+ .td().__(lastStartTime).__()
+ .td().__(lastHeartBeat).__()
+ .td(resources)
+ .td(nodes)
+ .__();
+
+ // Formatted memory information
+ long allocatedMB = subClusterMetricsInfo.getAllocatedMB();
+ String allocatedMBDesc = StringUtils.byteDesc(allocatedMB * BYTES_IN_MB);
+ long availableMB = subClusterMetricsInfo.getAvailableMB();
+ String availableMBDesc = StringUtils.byteDesc(availableMB * BYTES_IN_MB);
+ long pendingMB = subClusterMetricsInfo.getPendingMB();
+ String pendingMBDesc = StringUtils.byteDesc(pendingMB * BYTES_IN_MB);
+ long reservedMB = subClusterMetricsInfo.getReservedMB();
+ String reservedMBDesc = StringUtils.byteDesc(reservedMB * BYTES_IN_MB);
+
+ subClusterMap.put("totalmemory", totalMBDesc);
+ subClusterMap.put("allocatedmemory", allocatedMBDesc);
+ subClusterMap.put("availablememory", availableMBDesc);
+ subClusterMap.put("pendingmemory", pendingMBDesc);
+ subClusterMap.put("reservedmemory", reservedMBDesc);
+ subClusterMap.put("subcluster", subClusterId.getId());
+ subClusterMap.put("capability", capability);
+ lists.add(subClusterMap);
+ }
+
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java
index 1a157a10ce0..f27692b5ad5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/MetricsOverviewTable.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.yarn.server.router.webapp;
import com.google.inject.Inject;
import com.sun.jersey.api.client.Client;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
@@ -206,28 +208,53 @@ public class MetricsOverviewTable extends RouterBlock {
// If Federation mode is not enabled or there is currently no SubCluster
available,
// each column in the list should be displayed as N/A
- if (!isEnabled || subclusters == null || subclusters.isEmpty()) {
- fsMetricsScheduleTr.tr().
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE).
- td(UNAVAILABLE)
- .__();
- } else {
+ if (!isEnabled) {
+ initLocalClusterOverViewTable(fsMetricsScheduleTr);
+ } else if (subclusters != null && !subclusters.isEmpty()) {
initSubClusterOverViewTable(metrics, fsMetricsScheduleTr, subclusters);
+ } else {
+ showRouterSchedulerMetricsData(UNAVAILABLE, fsMetricsScheduleTr);
}
fsMetricsScheduleTr.__().__();
}
+ /**
+ * We display Scheduler information for local cluster.
+ *
+ * @param fsMetricsScheduleTr MetricsScheduleTr.
+ */
+ private void initLocalClusterOverViewTable(
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr) {
+ // configuration
+ Configuration config = this.router.getConfig();
+ Client client = RouterWebServiceUtil.createJerseyClient(config);
+ String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(config);
+
+ // Get the name of the local cluster.
+ String localClusterName = config.get(YarnConfiguration.RM_CLUSTER_ID,
UNAVAILABLE);
+ SchedulerOverviewInfo schedulerOverviewInfo =
+ getSchedulerOverviewInfo(webAppAddress, config, client);
+ if (schedulerOverviewInfo != null) {
+ RouterSchedulerMetrics rsMetrics =
+ new RouterSchedulerMetrics(localClusterName, schedulerOverviewInfo);
+ // Basic information
+ showRouterSchedulerMetricsData(rsMetrics, fsMetricsScheduleTr);
+ } else {
+ showRouterSchedulerMetricsData(localClusterName, fsMetricsScheduleTr);
+ }
+ }
+
+ /**
+ * We display Scheduler information for multiple subClusters.
+ *
+ * @param metrics RouterClusterMetrics.
+ * @param fsMetricsScheduleTr MetricsScheduleTr.
+ * @param subClusters subCluster list.
+ */
private void initSubClusterOverViewTable(RouterClusterMetrics metrics,
Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr,
- Collection<SubClusterInfo> subclusters) {
+ Collection<SubClusterInfo> subClusters) {
// configuration
Configuration config = this.router.getConfig();
@@ -235,30 +262,93 @@ public class MetricsOverviewTable extends RouterBlock {
Client client = RouterWebServiceUtil.createJerseyClient(config);
// Traverse all SubClusters to get cluster information.
- for (SubClusterInfo subcluster : subclusters) {
+ for (SubClusterInfo subcluster : subClusters) {
+ // We need to make sure subCluster is not null
+ if (subcluster != null && subcluster.getSubClusterId() != null) {
+ // Call the RM interface to obtain schedule information
+ String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) +
+ subcluster.getRMWebServiceAddress();
+ SchedulerOverviewInfo schedulerOverviewInfo =
+ getSchedulerOverviewInfo(webAppAddress, config, client);
- // Call the RM interface to obtain schedule information
- String webAppAddress = WebAppUtils.getHttpSchemePrefix(config) +
- subcluster.getRMWebServiceAddress();
+ // If schedulerOverviewInfo is not null,
+ // We will display information from rsMetrics, otherwise we will not
display information.
+ if (schedulerOverviewInfo != null) {
+ RouterSchedulerMetrics rsMetrics =
+ new RouterSchedulerMetrics(subcluster, metrics,
schedulerOverviewInfo);
+ // Basic information
+ showRouterSchedulerMetricsData(rsMetrics, fsMetricsScheduleTr);
+ }
+ }
+ }
- SchedulerOverviewInfo typeInfo = RouterWebServiceUtil
+ client.destroy();
+ }
+
+ /**
+ * Get SchedulerOverview information based on webAppAddress.
+ *
+ * @param webAppAddress webAppAddress.
+ * @param config configuration.
+ * @param client jersey Client.
+ * @return SchedulerOverviewInfo.
+ */
+ private SchedulerOverviewInfo getSchedulerOverviewInfo(String webAppAddress,
+ Configuration config, Client client) {
+ try {
+ SchedulerOverviewInfo schedulerOverviewInfo = RouterWebServiceUtil
.genericForward(webAppAddress, null, SchedulerOverviewInfo.class,
HTTPMethods.GET,
RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.SCHEDULER_OVERVIEW,
null, null,
- config, client);
- RouterSchedulerMetrics rsMetrics = new
RouterSchedulerMetrics(subcluster, metrics, typeInfo);
-
- // Basic information
- fsMetricsScheduleTr.tr().
- td(rsMetrics.getSubCluster()).
- td(rsMetrics.getSchedulerType()).
- td(rsMetrics.getSchedulingResourceType()).
- td(rsMetrics.getMinimumAllocation()).
- td(rsMetrics.getMaximumAllocation()).
- td(rsMetrics.getApplicationPriority()).
- td(rsMetrics.getSchedulerBusy()).
- td(rsMetrics.getRmDispatcherEventQueueSize()).
- td(rsMetrics.getSchedulerDispatcherEventQueueSize()).
- __();
+ config, client);
+ return schedulerOverviewInfo;
+ } catch (Exception e) {
+ LOG.error("get SchedulerOverviewInfo from webAppAddress = {} error.",
+ webAppAddress, e);
+ return null;
}
}
+
+ /**
+ * Show RouterSchedulerMetricsData.
+ *
+ * @param rsMetrics routerSchedulerMetrics.
+ * @param fsMetricsScheduleTr MetricsScheduleTr.
+ */
+ private void showRouterSchedulerMetricsData(RouterSchedulerMetrics rsMetrics,
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr) {
+ // Basic information
+ fsMetricsScheduleTr.tr().
+ td(rsMetrics.getSubCluster()).
+ td(rsMetrics.getSchedulerType()).
+ td(rsMetrics.getSchedulingResourceType()).
+ td(rsMetrics.getMinimumAllocation()).
+ td(rsMetrics.getMaximumAllocation()).
+ td(rsMetrics.getApplicationPriority()).
+ td(rsMetrics.getSchedulerBusy()).
+ td(rsMetrics.getRmDispatcherEventQueueSize()).
+ td(rsMetrics.getSchedulerDispatcherEventQueueSize()).
+ __();
+ }
+
+ /**
+ * Show RouterSchedulerMetricsData.
+ *
+ * @param subClusterId subClusterId.
+ * @param fsMetricsScheduleTr MetricsScheduleTr.
+ */
+ private void showRouterSchedulerMetricsData(String subClusterId,
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet.DIV<Hamlet>>> fsMetricsScheduleTr) {
+ String subCluster = StringUtils.isNotBlank(subClusterId) ? subClusterId :
UNAVAILABLE;
+ fsMetricsScheduleTr.tr().
+ td(subCluster).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE).
+ td(UNAVAILABLE)
+ .__();
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
index 2266370ee95..aa04096688f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NavBlock.java
@@ -40,11 +40,14 @@ public class NavBlock extends RouterBlock {
@Override
public void render(Block html) {
+
+ String federationText = isYarnFederationEnabled() ? "Federation" :
"LocalCluster";
+
Hamlet.UL<Hamlet.DIV<Hamlet>> mainList = html.div("#nav").
h3("Cluster").
ul().
li().a(url(""), "About").__().
- li().a(url("federation"), "Federation").__();
+ li().a(url("federation"), federationText).__();
List<String> subClusterIds = getActiveSubClusterIds();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java
index 4b77164bec7..37d1d3c95a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodeLabelsBlock.java
@@ -93,14 +93,34 @@ public class NodeLabelsBlock extends RouterBlock {
return null;
}
+ /**
+ * We will obtain the NodeLabel information of multiple sub-clusters.
+ *
+ * If Federation mode is enabled, get the NodeLabels of multiple
sub-clusters,
+ * otherwise get the NodeLabels of the local cluster.
+ *
+ * @param isEnabled Whether to enable Federation mode,
+ * true, Federation mode; false, Non-Federation mode.
+ *
+ * @return NodeLabelsInfo.
+ */
private NodeLabelsInfo getYarnFederationNodeLabelsInfo(boolean isEnabled) {
+ Configuration config = this.router.getConfig();
+ String webAddress;
if (isEnabled) {
- String webAddress =
WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig());
- return getSubClusterNodeLabelsByWebAddress(webAddress);
+ webAddress = WebAppUtils.getRouterWebAppURLWithScheme(config);
+ } else {
+ webAddress = WebAppUtils.getRMWebAppURLWithScheme(config);
}
- return null;
+ return getSubClusterNodeLabelsByWebAddress(webAddress);
}
+ /**
+ * Get NodeLabels based on WebAddress.
+ *
+ * @param webAddress RM WebAddress.
+ * @return NodeLabelsInfo.
+ */
private NodeLabelsInfo getSubClusterNodeLabelsByWebAddress(String
webAddress) {
Configuration conf = this.router.getConfig();
Client client = RouterWebServiceUtil.createJerseyClient(conf);
@@ -112,6 +132,12 @@ public class NodeLabelsBlock extends RouterBlock {
return nodes;
}
+ /**
+ * Initialize the Router page based on NodeLabels.
+ *
+ * @param nodeLabelsInfo NodeLabelsInfo.
+ * @param html html Block.
+ */
private void initYarnFederationNodeLabelsOfCluster(NodeLabelsInfo
nodeLabelsInfo, Block html) {
Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody = html.table("#nodelabels").
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
index 8d0fa5322b9..f5c0f9f9d1d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/NodesBlock.java
@@ -65,7 +65,7 @@ public class NodesBlock extends RouterBlock {
// We will try to get the subClusterName.
// If the subClusterName is not empty,
// it means that we need to get the Node list of a subCluster.
- NodesInfo nodesInfo = null;
+ NodesInfo nodesInfo;
if (subClusterName != null && !subClusterName.isEmpty()) {
initSubClusterMetricsOverviewTable(html, subClusterName);
nodesInfo = getSubClusterNodesInfo(subClusterName);
@@ -80,11 +80,14 @@ public class NodesBlock extends RouterBlock {
}
private NodesInfo getYarnFederationNodesInfo(boolean isEnabled) {
+ Configuration config = this.router.getConfig();
+ String webAddress;
if (isEnabled) {
- String webAddress =
WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig());
- return getSubClusterNodesInfoByWebAddress(webAddress);
+ webAddress =
WebAppUtils.getRouterWebAppURLWithScheme(this.router.getConfig());
+ } else {
+ webAddress = WebAppUtils.getRMWebAppURLWithScheme(config);
}
- return null;
+ return getSubClusterNodesInfoByWebAddress(webAddress);
}
private NodesInfo getSubClusterNodesInfo(String subCluster) {
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java
index 7fd8d246a33..c811c4d48c6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterBlock.java
@@ -18,21 +18,29 @@
package org.apache.hadoop.yarn.server.router.webapp;
import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import
org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWSConsts;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.router.Router;
import org.apache.hadoop.yarn.webapp.hamlet2.Hamlet;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import java.io.StringWriter;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
@@ -62,36 +70,58 @@ public abstract class RouterBlock extends HtmlBlock {
*/
protected ClusterMetricsInfo getRouterClusterMetricsInfo() {
boolean isEnabled = isYarnFederationEnabled();
+ String webAppAddress;
if(isEnabled) {
- String webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
- Client client = RouterWebServiceUtil.createJerseyClient(conf);
- ClusterMetricsInfo metrics = RouterWebServiceUtil
- .genericForward(webAppAddress, null, ClusterMetricsInfo.class,
HTTPMethods.GET,
- RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
- conf, client);
- client.destroy();
- return metrics;
+ webAppAddress = WebAppUtils.getRouterWebAppURLWithScheme(conf);
+ } else {
+ webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(conf);
}
- return null;
+ return getClusterMetricsInfo(webAppAddress);
+ }
+
+ /**
+ * Get RouterClusterMetrics Info.
+ *
+ * @param webAppAddress webAppAddress.
+ * @return ClusterMetricsInfo.
+ */
+ protected ClusterMetricsInfo getClusterMetricsInfo(String webAppAddress) {
+ // If webAppAddress is empty, we will return NULL.
+ if (StringUtils.isBlank(webAppAddress)) {
+ return null;
+ }
+
+ // We will get ClusterMetricsInfo By webAppAddress.
+ Client client = RouterWebServiceUtil.createJerseyClient(conf);
+ ClusterMetricsInfo metrics = RouterWebServiceUtil
+ .genericForward(webAppAddress, null, ClusterMetricsInfo.class,
HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
+ conf, client);
+ client.destroy();
+ return metrics;
}
/**
* Get a list of subclusters.
*
* @return subcluster List.
- * @throws YarnException if the call to the getSubClusters is unsuccessful.
*/
- protected List<SubClusterInfo> getSubClusterInfoList() throws YarnException {
-
- Map<SubClusterId, SubClusterInfo> subClustersInfo =
facade.getSubClusters(true);
+ protected List<SubClusterInfo> getSubClusterInfoList() {
+ List<SubClusterInfo> subClusters = new ArrayList<>();
+ try {
+ Map<SubClusterId, SubClusterInfo> subClustersInfo =
facade.getSubClusters(true);
- // Sort the SubClusters.
- List<SubClusterInfo> subclusters = new ArrayList<>();
- subclusters.addAll(subClustersInfo.values());
- Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o ->
o.getSubClusterId());
- Collections.sort(subclusters, cmp);
+ // Sort the SubClusters.
+ subClusters.addAll(subClustersInfo.values());
+ Comparator<? super SubClusterInfo> cmp = Comparator.comparing(o ->
o.getSubClusterId());
+ Collections.sort(subClusters, cmp);
- return subclusters;
+ // Return results
+ return subClusters;
+ } catch (YarnException e) {
+ LOG.error("getSubClusterInfoList error.", e);
+ return subClusters;
+ }
}
/**
@@ -163,6 +193,12 @@ public abstract class RouterBlock extends HtmlBlock {
return null;
}
+ /**
+ * Get SubClusterInfo based on subclusterId.
+ *
+ * @param subclusterId subCluster Id
+ * @return SubClusterInfo Collection
+ */
protected Collection<SubClusterInfo> getSubClusterInfoList(String
subclusterId) {
try {
SubClusterId subClusterId = SubClusterId.newInstance(subclusterId);
@@ -257,4 +293,63 @@ public abstract class RouterBlock extends HtmlBlock {
mainList.li().a(url("nodelabels"), "Node Labels").__();
}
}
+
+ /**
+ * Generate SubClusterInfo based on local cluster information.
+ *
+ * @param config Configuration.
+ * @return SubClusterInfo.
+ */
+ protected SubClusterInfo getSubClusterInfoByLocalCluster(Configuration
config) {
+
+ Client client = null;
+ try {
+
+ // Step1. Retrieve the name of the local cluster and ClusterMetricsInfo.
+ String localClusterName = config.get(YarnConfiguration.RM_CLUSTER_ID,
UNAVAILABLE);
+ String webAppAddress = WebAppUtils.getRMWebAppURLWithScheme(config);
+ String rmWebAppURLWithoutScheme =
WebAppUtils.getRMWebAppURLWithoutScheme(config);
+ client = RouterWebServiceUtil.createJerseyClient(config);
+ ClusterMetricsInfo clusterMetricsInfos = RouterWebServiceUtil
+ .genericForward(webAppAddress, null, ClusterMetricsInfo.class,
HTTPMethods.GET,
+ RMWSConsts.RM_WEB_SERVICE_PATH + RMWSConsts.METRICS, null, null,
+ config, client);
+
+ if (clusterMetricsInfos == null) {
+ return null;
+ }
+
+ // Step2. Retrieve cluster information for the local cluster to obtain
its startup time.
+ ClusterInfo clusterInfo =
RouterWebServiceUtil.genericForward(webAppAddress, null,
+ ClusterInfo.class, HTTPMethods.GET, RMWSConsts.RM_WEB_SERVICE_PATH +
RMWSConsts.INFO,
+ null, null, config, client);
+
+ if (clusterInfo == null) {
+ return null;
+ }
+
+ // Step3. Get Local-Cluster Capability
+ JSONJAXBContext jc = new JSONJAXBContext(
+ JSONConfiguration.mapped().rootUnwrapping(false).build(),
ClusterMetricsInfo.class);
+ JSONMarshaller marshaller = jc.createJSONMarshaller();
+ StringWriter writer = new StringWriter();
+ marshaller.marshallToJSON(clusterMetricsInfos, writer);
+ String capability = writer.toString();
+
+ // Step4. Generate SubClusterInfo.
+ SubClusterId subClusterId = SubClusterId.newInstance(localClusterName);
+ SubClusterInfo subClusterInfo = SubClusterInfo.newInstance(subClusterId,
+ rmWebAppURLWithoutScheme, SubClusterState.SC_RUNNING,
clusterInfo.getStartedOn(),
+ Time.now(), capability);
+
+ return subClusterInfo;
+ } catch (Exception e) {
+ LOG.error("An error occurred while parsing the local YARN cluster.", e);
+ } finally {
+ if (client != null) {
+ client.destroy();
+ }
+ }
+ return null;
+ }
}
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java
index 4a3af1ba43d..714642cec83 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/dao/RouterSchedulerMetrics.java
@@ -51,9 +51,20 @@ public class RouterSchedulerMetrics {
public RouterSchedulerMetrics(SubClusterInfo subClusterInfo,
RouterClusterMetrics metrics,
SchedulerOverviewInfo overview) {
+ if (subClusterInfo != null) {
+ initRouterSchedulerMetrics(subClusterInfo.getSubClusterId().getId(),
overview);
+ }
+ }
+
+ public RouterSchedulerMetrics(String localClusterName, SchedulerOverviewInfo
overview) {
+ initRouterSchedulerMetrics(localClusterName, overview);
+ }
+
+ private void initRouterSchedulerMetrics(String subClusterName,
+ SchedulerOverviewInfo overview) {
try {
// Parse Scheduler Information.
- this.subCluster = subClusterInfo.getSubClusterId().getId();
+ this.subCluster = subClusterName;
this.schedulerType = overview.getSchedulerType();
this.schedulingResourceType = overview.getSchedulingResourceType();
this.minimumAllocation = overview.getMinimumAllocation().toString();
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
index f1501fe1e7a..f703dab9552 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationWebApp.java
@@ -116,6 +116,7 @@ public class TestFederationWebApp extends
TestRouterWebServicesREST {
@Test
public void testNodeLabelAppViewNotEnable()
throws InterruptedException, YarnException, IOException {
+ LOG.info("testNodeLabelAppViewNotEnable - NotEnable Federation.");
// Test Federation Not Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, false);
@@ -125,6 +126,7 @@ public class TestFederationWebApp extends
TestRouterWebServicesREST {
@Test
public void testNodeLabelAppViewEnable()
throws InterruptedException, YarnException, IOException {
+ LOG.info("testNodeLabelAppViewEnable - Enable Federation.");
// Test Federation Not Enabled
Configuration config = new YarnConfiguration();
config.setBoolean(YarnConfiguration.FEDERATION_ENABLED, true);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
index 55461680b5b..4a28627a9a1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
@@ -43,4 +43,8 @@
<name>yarn.federation.policy-manager-params</name>
<value>{"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"}</value>
</property>
+ <property>
+ <name>yarn.resourcemanager.cluster-id</name>
+ <value>local-cluster</value>
+ </property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org
For additional commands, e-mail: common-commits-h...@hadoop.apache.org
