goiri commented on code in PR #4757:
URL: https://github.com/apache/hadoop/pull/4757#discussion_r949711369
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationStatisticsInfo.java:
##########
@@ -33,12 +34,15 @@ public class ApplicationStatisticsInfo {
public ApplicationStatisticsInfo() {
} // JAXB needs this
+ public ApplicationStatisticsInfo(Collection<StatisticsItemInfo> items) {
+ statItem.addAll(items);
+ }
+
public void add(StatisticsItemInfo statItem) {
this.statItem.add(statItem);
}
public ArrayList<StatisticsItemInfo> getStatItems() {
return statItem;
}
-
Review Comment:
Avoid
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java:
##########
@@ -540,4 +542,34 @@ public static NodeToLabelsInfo mergeNodeToLabels(
return new NodeToLabelsInfo(nodeToLabels);
}
+
+ public static ApplicationStatisticsInfo mergeApplicationStatisticsInfo(
+ Collection<ApplicationStatisticsInfo> appStatistics) {
+ ApplicationStatisticsInfo result = new ApplicationStatisticsInfo();
+ HashMap<String, StatisticsItemInfo> statisticsItemMap = new HashMap();
+
+ appStatistics.stream().forEach(appStatistic -> {
+ List<StatisticsItemInfo> statisticsItemInfos =
appStatistic.getStatItems();
+ for (StatisticsItemInfo statisticsItemInfo : statisticsItemInfos) {
+ String statisticsItemKey = statisticsItemInfo.getType() + "_" +
statisticsItemInfo.getState().toString();
+ StatisticsItemInfo statisticsItemValue =
+ statisticsItemMap.getOrDefault(statisticsItemKey, null);
+ if (statisticsItemValue != null) {
Review Comment:
Cleaner to do contains or similar.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java:
##########
@@ -592,4 +595,90 @@ public static AppAttemptInfo generateAppAttemptInfo(int
attemptId) {
when(appAttemptInfo.getLogsLink()).thenReturn("LogLink_" + attemptId);
return appAttemptInfo;
}
+
+ @Test
+ public void testMergeApplicationStatisticsInfo() {
+ ApplicationStatisticsInfo infoA = new ApplicationStatisticsInfo();
+ ApplicationStatisticsInfo infoB = new ApplicationStatisticsInfo();
+
+ StatisticsItemInfo item1 =
+ new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 10);
+ StatisticsItemInfo item2 =
+ new StatisticsItemInfo(YarnApplicationState.ACCEPTED, "*", 20);
+
+ infoA.add(item1);
+ infoB.add(item2);
+
+ List<ApplicationStatisticsInfo> lists = new ArrayList<>();
+ lists.add(infoA);
+ lists.add(infoB);
+
+ ApplicationStatisticsInfo mergeInfo =
+ RouterWebServiceUtil.mergeApplicationStatisticsInfo(lists);
+
+ Assert.assertEquals(1, mergeInfo.getStatItems().size());
+ Assert.assertEquals(item1.getCount() + item2.getCount(),
+ mergeInfo.getStatItems().get(0).getCount());
Review Comment:
extract get(0) and check it first.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java:
##########
@@ -661,4 +683,108 @@ public Response updateAppQueue(AppQueue targetQueue,
HttpServletRequest hsr, Str
AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
return Response.status(Status.OK).entity(targetAppQueue).build();
}
+
+ public void updateApplicationState(YarnApplicationState appState, String
appId)
+ throws AuthorizationException, YarnException, InterruptedException,
IOException {
+ validateRunning();
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ ApplicationReport appReport = applicationMap.get(applicationId);
+ appReport.setYarnApplicationState(appState);
+ }
+
+ @Override
+ public ApplicationStatisticsInfo getAppStatistics(
+ HttpServletRequest hsr, Set<String> stateQueries, Set<String>
typeQueries) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+ for (HashMap.Entry<ApplicationId, ApplicationReport> item :
applicationMap.entrySet()) {
+
+ ApplicationReport applicationReport = item.getValue();
Review Comment:
If we don't do getKey, we can iterate .values()
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java:
##########
@@ -1129,13 +1129,49 @@ public AppActivitiesInfo
getAppActivities(HttpServletRequest hsr,
String appId, String time, Set<String> requestPriorities,
Set<String> allocationRequestIds, String groupBy, String limit,
Set<String> actions, boolean summarize) {
- throw new NotImplementedException("Code is not implemented");
+
+ // Only verify the app_id, because the specific subCluster needs to be
found according to the app_id,
+ // and other verifications are directly handed over to the corresponding
subCluster RM
+ if (appId == null || appId.isEmpty()) {
+ throw new IllegalArgumentException("Parameter error, the appId is empty
or null.");
+ }
+
+ try {
+ SubClusterInfo subClusterInfo = getHomeSubClusterInfoByAppId(appId);
+ DefaultRequestInterceptorREST interceptor =
getOrCreateInterceptorForSubCluster(
+ subClusterInfo.getSubClusterId(),
subClusterInfo.getRMWebServiceAddress());
+
+ final HttpServletRequest hsrCopy = clone(hsr);
+ return interceptor.getAppActivities(hsrCopy, appId, time,
requestPriorities,
+ allocationRequestIds, groupBy, limit, actions, summarize);
+ } catch (IllegalArgumentException e) {
+ RouterServerUtil.logAndThrowRunTimeException(e,
Review Comment:
Single line?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java:
##########
@@ -661,4 +683,108 @@ public Response updateAppQueue(AppQueue targetQueue,
HttpServletRequest hsr, Str
AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
return Response.status(Status.OK).entity(targetAppQueue).build();
}
+
+ public void updateApplicationState(YarnApplicationState appState, String
appId)
+ throws AuthorizationException, YarnException, InterruptedException,
IOException {
+ validateRunning();
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ ApplicationReport appReport = applicationMap.get(applicationId);
+ appReport.setYarnApplicationState(appState);
+ }
+
+ @Override
+ public ApplicationStatisticsInfo getAppStatistics(
+ HttpServletRequest hsr, Set<String> stateQueries, Set<String>
typeQueries) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+ for (HashMap.Entry<ApplicationId, ApplicationReport> item :
applicationMap.entrySet()) {
+
+ ApplicationReport applicationReport = item.getValue();
+ YarnApplicationState appState =
applicationReport.getYarnApplicationState();
+ String appType = applicationReport.getApplicationType();
+
+ if (stateQueries.contains(appState.name()) &&
typeQueries.contains(appType)) {
+ String itemInfoMapKey = appState.toString() + "_" + appType;
+ StatisticsItemInfo itemInfo = itemInfoMap.getOrDefault(itemInfoMapKey,
null);
+ if (itemInfo == null) {
+ itemInfo = new StatisticsItemInfo(appState, appType, 1);
+ } else {
+ long newCount = itemInfo.getCount() + 1;
+ itemInfo.setCount(newCount);
+ }
+ itemInfoMap.put(itemInfoMapKey, itemInfo);
+ }
+ }
+
+ ArrayList<StatisticsItemInfo> itemInfos = new
ArrayList<>(itemInfoMap.values());
+
+ return new ApplicationStatisticsInfo(itemInfos);
Review Comment:
You could do:
```
return new ApplicationStatisticsInfo(itemInfoMap.values());
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/MockDefaultRequestInterceptorREST.java:
##########
@@ -661,4 +683,108 @@ public Response updateAppQueue(AppQueue targetQueue,
HttpServletRequest hsr, Str
AppQueue targetAppQueue = new AppQueue(targetQueue.getQueue());
return Response.status(Status.OK).entity(targetAppQueue).build();
}
+
+ public void updateApplicationState(YarnApplicationState appState, String
appId)
+ throws AuthorizationException, YarnException, InterruptedException,
IOException {
+ validateRunning();
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+ ApplicationReport appReport = applicationMap.get(applicationId);
+ appReport.setYarnApplicationState(appState);
+ }
+
+ @Override
+ public ApplicationStatisticsInfo getAppStatistics(
+ HttpServletRequest hsr, Set<String> stateQueries, Set<String>
typeQueries) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ Map<String, StatisticsItemInfo> itemInfoMap = new HashMap<>();
+
+ for (HashMap.Entry<ApplicationId, ApplicationReport> item :
applicationMap.entrySet()) {
+
+ ApplicationReport applicationReport = item.getValue();
+ YarnApplicationState appState =
applicationReport.getYarnApplicationState();
+ String appType = applicationReport.getApplicationType();
+
+ if (stateQueries.contains(appState.name()) &&
typeQueries.contains(appType)) {
+ String itemInfoMapKey = appState.toString() + "_" + appType;
+ StatisticsItemInfo itemInfo = itemInfoMap.getOrDefault(itemInfoMapKey,
null);
+ if (itemInfo == null) {
+ itemInfo = new StatisticsItemInfo(appState, appType, 1);
+ } else {
+ long newCount = itemInfo.getCount() + 1;
+ itemInfo.setCount(newCount);
+ }
+ itemInfoMap.put(itemInfoMapKey, itemInfo);
+ }
+ }
+
+ ArrayList<StatisticsItemInfo> itemInfos = new
ArrayList<>(itemInfoMap.values());
+
+ return new ApplicationStatisticsInfo(itemInfos);
+ }
+
+ @Override
+ public AppActivitiesInfo getAppActivities(
+ HttpServletRequest hsr, String appId, String time, Set<String>
requestPriorities,
+ Set<String> allocationRequestIds, String groupBy, String limit,
Set<String> actions,
+ boolean summarize) {
+ if (!isRunning) {
+ throw new RuntimeException("RM is stopped");
+ }
+
+ ApplicationId applicationId = ApplicationId.fromString(appId);
+ if (!applicationMap.containsKey(applicationId)) {
+ throw new NotFoundException("app with id: " + appId + " not found");
+ }
+
+ SchedulerNode schedulerNode =
Review Comment:
Single line
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
