[
https://issues.apache.org/jira/browse/YARN-11219?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17581585#comment-17581585
]
ASF GitHub Bot commented on YARN-11219:
---------------------------------------
goiri commented on code in PR #4757:
URL: https://github.com/apache/hadoop/pull/4757#discussion_r949711369
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java:
##########
@@ -33,12 +34,15 @@ public class ApplicationStatisticsInfo {
public ApplicationStatisticsInfo() {
} // JAXB needs this
+ public ApplicationStatisticsInfo(Collection<StatisticsItemInfo> items) {
+ statItem.addAll(items);
+ }
+
public void add(StatisticsItemInfo statItem) {
this.statItem.add(statItem);
}
public ArrayList<StatisticsItemInfo> getStatItems() {
return statItem;
}
-
Review Comment:
Avoid
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java:
##########
@@ -540,4 +542,34 @@ public static NodeToLabelsInfo mergeNodeToLabels(
return new NodeToLabelsInfo(nodeToLabels);
}
+
+ public static ApplicationStatisticsInfo mergeApplicationStatisticsInfo(
+ Collection<ApplicationStatisticsInfo> appStatistics) {
+ ApplicationStatisticsInfo result = new ApplicationStatisticsInfo();
+ HashMap<String, StatisticsItemInfo> statisticsItemMap = new HashMap();
+
+ appStatistics.stream().forEach(appStatistic -> {
+ List<StatisticsItemInfo> statisticsItemInfos =
appStatistic.getStatItems();
+ for (StatisticsItemInfo statisticsItemInfo : statisticsItemInfos) {
+ String statisticsItemKey = statisticsItemInfo.getType() + "_" +
statisticsItemInfo.getState().toString();
+ StatisticsItemInfo statisticsItemValue =
+ statisticsItemMap.getOrDefault(statisticsItemKey, null);
+ if (statisticsItemValue != null) {
Review Comment:
Cleaner to do contains or similar.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java:
##########
@@ -592,4 +595,90 @@ public static AppAttemptInfo generateAppAttemptInfo(int
attemptId) {
when(appAttemptInfo.getLogsLink()).thenReturn("LogLink_" + attemptId);
return appAttemptInfo;
}
+
+ @Test
+ public void testMergeApplicationStatisticsInfo() {
+ ApplicationStatisticsInfo infoA = new ApplicationStatisticsInfo();
+ ApplicationStatisticsInfo infoB = new ApplicationStatisticsInfo();
+
+ StatisticsItemInfo item1 =
+ new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 10);
+ StatisticsItemInfo item2 =
+ new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 20);
+
+ infoA.add(item1);
+ infoB.add(item2);
+
+ List<ApplicationStatisticsInfo> lists = new ArrayList<>();
+ lists.add(infoA);
+ lists.add(infoB);
+
+ ApplicationStatisticsInfo mergeInfo =
+ RouterWebServiceUtil.mergeApplicationStatisticsInfo(lists);
+
+ Assert.assertEquals(1, mergeInfo.getStatItems().size());
+ Assert.assertEquals(item1.getCount() + item2.getCount(),
+ mergeInfo.getStatItems().get(0).getCount());
Review Comment:
extract get(0) and check it first.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java:
##########
@@ -661,4 +683,108 @@ public Response updateAppQueue(AppQueue targetQueue,
HttpServletRequest hsr, Str
AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
return Response.status(Status.OK).entity(targetAppQueue).build();
}
+
+ public void updateApplicationState(YarnApplicationState appState, String
appId)
+ throws AuthorizationException, YarnException, InterruptedException,
IOException {
+ validateRunning();
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ ApplicationReport appReport = applicationMap.get(applicationId);
+ appReport.setYarnApplicationState(appState);
+ }
+
+ @Override
+ public ApplicationStatisticsInfo getAppStatistics(
+ HttpServletRequest hsr, Set<String> stateQueries, Set<String>
typeQueries) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+ for (HashMap.Entry<ApplicationId, ApplicationReport> item :
applicationMap.entrySet()) {
+
+ ApplicationReport applicationReport = item.getValue();
Review Comment:
If we don't do getKey, we can iterate .values()
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java:
##########
@@ -1129,13 +1129,49 @@ public AppActivitiesInfo
getAppActivities(HttpServletRequest hsr,
String appId, String time, Set<String> requestPriorities,
Set<String> allocationRequestIds, String groupBy, String limit,
Set<String> actions, boolean summarize) {
- throw new NotImplementedException("Code is not implemented");
+
+ // Only verify the app_id, because the specific subCluster needs to be
found according to the app_id,
+ // and other verifications are directly handed over to the corresponding
subCluster RM
+ if (appId == null || appId.isEmpty()) {
+ throw new IllegalArgumentException("Parameter error, the appId is empty
or null.");
+ }
+
+ try {
+ SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+ DefaultRequestInterceptorREST interceptor =
getOrCreateInterceptorForSubCluster(
+ subClusterInfo.getSubClusterId(),
subClusterInfo.getRMWebServiceAddress());
+
+ final HttpServletRequest hsrCopy = clone(hsr);
+ return interceptor.getAppActivities(hsrCopy, appId, time,
requestPriorities,
+ allocationRequestIds, groupBy, limit, actions, summarize);
+ } catch (IllegalArgumentException e) {
+ RouterServerUtil.logAndThrowRunTimeException(e,
Review Comment:
Single line?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java:
##########
@@ -661,4 +683,108 @@ public Response updateAppQueue(AppQueue targetQueue,
HttpServletRequest hsr, Str
AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
return Response.status(Status.OK).entity(targetAppQueue).build();
}
+
+ public void updateApplicationState(YarnApplicationState appState, String
appId)
+ throws AuthorizationException, YarnException, InterruptedException,
IOException {
+ validateRunning();
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ ApplicationReport appReport = applicationMap.get(applicationId);
+ appReport.setYarnApplicationState(appState);
+ }
+
+ @Override
+ public ApplicationStatisticsInfo getAppStatistics(
+ HttpServletRequest hsr, Set<String> stateQueries, Set<String>
typeQueries) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+ for (HashMap.Entry<ApplicationId, ApplicationReport> item :
applicationMap.entrySet()) {
+
+ ApplicationReport applicationReport = item.getValue();
+ YarnApplicationState appState =
applicationReport.getYarnApplicationState();
+ String appType = applicationReport.getApplicationType();
+
+ if (stateQueries.contains(appState.name()) &&
typeQueries.contains(appType)) {
+ String itemInfoMapKey = appState.toString() + "_" + appType;
+ StatisticsItemInfo itemInfo = itemInfoMap.getOrDefault(itemInfoMapKey,
null);
+ if (itemInfo == null) {
+ itemInfo = new StatisticsItemInfo(appState, appType, 1);
+ } else {
+ long newCount = itemInfo.getCount() + 1;
+ itemInfo.setCount(newCount);
+ }
+ itemInfoMap.put(itemInfoMapKey, itemInfo);
+ }
+ }
+
+ ArrayList<StatisticsItemInfo> itemInfos = new
ArrayList<>(itemInfoMap.values());
+
+ return new ApplicationStatisticsInfo(itemInfos);
Review Comment:
You could do:
```
return new ApplicationStatisticsInfo(itemInfoMap.values());
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java:
##########
@@ -661,4 +683,108 @@ public Response updateAppQueue(AppQueue targetQueue,
HttpServletRequest hsr, Str
AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
return Response.status(Status.OK).entity(targetAppQueue).build();
}
+
+ public void updateApplicationState(YarnApplicationState appState, String
appId)
+ throws AuthorizationException, YarnException, InterruptedException,
IOException {
+ validateRunning();
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ ApplicationReport appReport = applicationMap.get(applicationId);
+ appReport.setYarnApplicationState(appState);
+ }
+
+ @Override
+ public ApplicationStatisticsInfo getAppStatistics(
+ HttpServletRequest hsr, Set<String> stateQueries, Set<String>
typeQueries) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+ for (HashMap.Entry<ApplicationId, ApplicationReport> item :
applicationMap.entrySet()) {
+
+ ApplicationReport applicationReport = item.getValue();
+ YarnApplicationState appState =
applicationReport.getYarnApplicationState();
+ String appType = applicationReport.getApplicationType();
+
+ if (stateQueries.contains(appState.name()) &&
typeQueries.contains(appType)) {
+ String itemInfoMapKey = appState.toString() + "_" + appType;
+ StatisticsItemInfo itemInfo = itemInfoMap.getOrDefault(itemInfoMapKey,
null);
+ if (itemInfo == null) {
+ itemInfo = new StatisticsItemInfo(appState, appType, 1);
+ } else {
+ long newCount = itemInfo.getCount() + 1;
+ itemInfo.setCount(newCount);
+ }
+ itemInfoMap.put(itemInfoMapKey, itemInfo);
+ }
+ }
+
+ ArrayList<StatisticsItemInfo> itemInfos = new
ArrayList<>(itemInfoMap.values());
+
+ return new ApplicationStatisticsInfo(itemInfos);
+ }
+
+ @Override
+ public AppActivitiesInfo getAppActivities(
+ HttpServletRequest hsr, String appId, String time, Set<String>
requestPriorities,
+ Set<String> allocationRequestIds, String groupBy, String limit,
Set<String> actions,
+ boolean summarize) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+
+ SchedulerNode schedulerNode =
Review Comment:
Single line
> [Federation] Add getAppActivities, getAppStatistics REST APIs for Router
> ------------------------------------------------------------------------
>
> Key: YARN-11219
> URL: https://issues.apache.org/jira/browse/YARN-11219
> Project: Hadoop YARN
> Issue Type: Sub-task
> Components: federation
> Affects Versions: 3.4.0
> Reporter: fanshilun
> Assignee: fanshilun
> Priority: Major
> Labels: pull-request-available
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
