Repository: hadoop Updated Branches: refs/heads/trunk 8571507ef -> ee3355be3
YARN-8303. YarnClient should contact TimelineReader for application/attempt/container report. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee3355be Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee3355be Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee3355be Branch: refs/heads/trunk Commit: ee3355be3cd1cd956296a1b79b264177ff139ce6 Parents: 8571507 Author: Rohith Sharma K S <rohithsharm...@apache.org> Authored: Fri Nov 16 17:51:40 2018 +0530 Committer: Rohith Sharma K S <rohithsharm...@apache.org> Committed: Fri Nov 16 17:58:19 2018 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/client/api/AHSClient.java | 10 +- .../yarn/client/api/impl/AHSv2ClientImpl.java | 149 ++++++ .../yarn/client/api/impl/YarnClientImpl.java | 194 +++++--- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 13 +- .../client/api/impl/TestAHSv2ClientImpl.java | 240 ++++++++++ .../hadoop/yarn/client/cli/TestLogsCLI.java | 3 + .../metrics/AppAttemptMetricsConstants.java | 69 +++ .../metrics/ApplicationMetricsConstants.java | 122 +++++ .../metrics/ContainerMetricsConstants.java | 83 ++++ .../yarn/server/metrics/package-info.java | 22 + .../timeline/TimelineEntityV2Converter.java | 449 +++++++++++++++++++ .../metrics/AppAttemptMetricsConstants.java | 69 --- .../metrics/ApplicationMetricsConstants.java | 122 ----- .../metrics/ContainerMetricsConstants.java | 83 ---- 14 files changed, 1288 insertions(+), 340 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java index b590a51..6da8e66 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AHSClient.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.client.api.impl.AHSClientImpl; +import org.apache.hadoop.yarn.client.api.impl.AHSv2ClientImpl; import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -46,8 +47,13 @@ public abstract class AHSClient extends AbstractService { */ @Public public static AHSClient createAHSClient() { - AHSClient client = new AHSClientImpl(); - return client; + return new AHSClientImpl(); + } + + @InterfaceStability.Evolving + @Public + public static AHSClient createAHSv2Client() { + return new AHSv2ClientImpl(); } @Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java new file mode 100644 index 0000000..e797c28 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AHSv2ClientImpl.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.client.api.AHSClient; +import org.apache.hadoop.yarn.client.api.TimelineReaderClient; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.util.timeline.TimelineEntityV2Converter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This class provides Application History client implementation which uses + * ATS v2 as backend. + */ +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class AHSv2ClientImpl extends AHSClient { + private TimelineReaderClient readerClient; + + public AHSv2ClientImpl() { + super(AHSv2ClientImpl.class.getName()); + } + + @Override + public void serviceInit(Configuration conf) { + readerClient = TimelineReaderClient.createTimelineReaderClient(); + readerClient.init(conf); + } + + @VisibleForTesting + protected void setReaderClient(TimelineReaderClient readerClient) { + this.readerClient = readerClient; + } + + @Override + public void serviceStart() { + readerClient.start(); + } + + @Override + public void serviceStop() { + readerClient.stop(); + } + + @Override + public ApplicationReport getApplicationReport(ApplicationId appId) + throws YarnException, IOException { + TimelineEntity entity = readerClient.getApplicationEntity( + appId, "ALL", null); + return TimelineEntityV2Converter.convertToApplicationReport(entity); + } + + @Override + public List<ApplicationReport> getApplications() + throws YarnException, IOException { + throw new UnsupportedOperationException("ATSv2.0 doesn't support retrieving" + + " ALL application entities."); + } + + @Override + public ApplicationAttemptReport getApplicationAttemptReport( + ApplicationAttemptId applicationAttemptId) + throws YarnException, IOException { + TimelineEntity entity = readerClient.getApplicationAttemptEntity( + applicationAttemptId, "ALL", null); + return TimelineEntityV2Converter.convertToApplicationAttemptReport(entity); + } + + @Override + public List<ApplicationAttemptReport> getApplicationAttempts( + ApplicationId applicationId) throws YarnException, IOException { + List<TimelineEntity> entities = readerClient.getApplicationAttemptEntities( + applicationId, "ALL", null, 0, null); + List<ApplicationAttemptReport> appAttemptReports = + new ArrayList<>(); + if (entities != null && !entities.isEmpty()) { + for (TimelineEntity entity : entities) { + ApplicationAttemptReport container = + TimelineEntityV2Converter.convertToApplicationAttemptReport( + entity); + appAttemptReports.add(container); + } + } + return appAttemptReports; + } + + @Override + public ContainerReport getContainerReport(ContainerId containerId) + throws YarnException, IOException { + TimelineEntity entity = readerClient.getContainerEntity(containerId, + "ALL", null); + return TimelineEntityV2Converter.convertToContainerReport(entity); + } + + @Override + public List<ContainerReport> getContainers(ApplicationAttemptId + applicationAttemptId) throws YarnException, IOException { + ApplicationId appId = applicationAttemptId.getApplicationId(); + Map<String, String> filters = new HashMap<>(); + filters.put("infofilters", "SYSTEM_INFO_PARENT_ENTITY eq {\"id\":\"" + + applicationAttemptId.toString() + + "\",\"type\":\"YARN_APPLICATION_ATTEMPT\"}"); + List<TimelineEntity> entities = readerClient.getContainerEntities( + appId, "ALL", filters, 0, null); + List<ContainerReport> containers = + new ArrayList<>(); + if (entities != null && !entities.isEmpty()) { + for (TimelineEntity entity : entities) { + ContainerReport container = + TimelineEntityV2Converter.convertToContainerReport( + entity); + containers.add(container); + } + } + return containers; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java index acfc3ff..227f7ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java @@ -149,6 +149,7 @@ public class YarnClientImpl extends YarnClient { private long asyncApiPollIntervalMillis; private long asyncApiPollTimeoutMillis; protected AHSClient historyClient; + private AHSClient ahsV2Client; private boolean historyServiceEnabled; protected volatile TimelineClient timelineClient; @VisibleForTesting @@ -159,6 +160,8 @@ public class YarnClientImpl extends YarnClient { protected boolean timelineServiceBestEffort; private boolean loadResourceTypesFromServer; + private boolean timelineV2ServiceEnabled; + private static final String ROOT = "root"; public YarnClientImpl() { @@ -188,6 +191,10 @@ public class YarnClientImpl extends YarnClient { timelineService = TimelineUtils.buildTimelineTokenService(conf); } + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + timelineV2ServiceEnabled = true; + } + // The AHSClientService is enabled by default when we start the // TimelineServer which means we are able to get history information // for applications/applicationAttempts/containers by using ahsClient @@ -200,6 +207,11 @@ public class YarnClientImpl extends YarnClient { historyClient.init(conf); } + if (timelineV2ServiceEnabled) { + ahsV2Client = AHSClient.createAHSv2Client(); + ahsV2Client.init(conf); + } + timelineServiceBestEffort = conf.getBoolean( YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT); @@ -223,6 +235,9 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.start(); } + if (timelineV2ServiceEnabled) { + ahsV2Client.start(); + } } catch (IOException e) { throw new YarnRuntimeException(e); } @@ -244,6 +259,9 @@ public class YarnClientImpl extends YarnClient { if (historyServiceEnabled) { historyClient.stop(); } + if (timelineV2ServiceEnabled) { + ahsV2Client.stop(); + } if (timelineClient != null) { timelineClient.stop(); } @@ -516,6 +534,14 @@ public class YarnClientImpl extends YarnClient { request.setApplicationId(appId); response = rmClient.getApplicationReport(request); } catch (ApplicationNotFoundException e) { + if (timelineV2ServiceEnabled) { + try { + return ahsV2Client.getApplicationReport(appId); + } catch (Exception ex) { + LOG.warn("Failed to fetch application report from " + + "ATS v2", ex); + } + } if (!historyServiceEnabled) { // Just throw it as usual if historyService is not enabled. throw e; @@ -726,15 +752,24 @@ public class YarnClientImpl extends YarnClient { .getApplicationAttemptReport(request); return response.getApplicationAttemptReport(); } catch (YarnException e) { - if (!historyServiceEnabled) { - // Just throw it as usual if historyService is not enabled. - throw e; - } + // Even if history-service is enabled, treat all exceptions still the same // except the following if (e.getClass() != ApplicationNotFoundException.class) { throw e; } + if (timelineV2ServiceEnabled) { + try { + return ahsV2Client.getApplicationAttemptReport(appAttemptId); + } catch (Exception ex) { + LOG.warn("Failed to fetch application attempt report from " + + "ATS v2", ex); + } + } + if (!historyServiceEnabled) { + // Just throw it as usual if historyService is not enabled. + throw e; + } return historyClient.getApplicationAttemptReport(appAttemptId); } } @@ -750,15 +785,23 @@ public class YarnClientImpl extends YarnClient { .getApplicationAttempts(request); return response.getApplicationAttemptList(); } catch (YarnException e) { - if (!historyServiceEnabled) { - // Just throw it as usual if historyService is not enabled. - throw e; - } // Even if history-service is enabled, treat all exceptions still the same // except the following if (e.getClass() != ApplicationNotFoundException.class) { throw e; } + if (timelineV2ServiceEnabled) { + try { + return ahsV2Client.getApplicationAttempts(appId); + } catch (Exception ex) { + LOG.warn("Failed to fetch application attempts from " + + "ATS v2", ex); + } + } + if (!historyServiceEnabled) { + // Just throw it as usual if historyService is not enabled. + throw e; + } return historyClient.getApplicationAttempts(appId); } } @@ -774,16 +817,24 @@ public class YarnClientImpl extends YarnClient { .getContainerReport(request); return response.getContainerReport(); } catch (YarnException e) { - if (!historyServiceEnabled) { - // Just throw it as usual if historyService is not enabled. - throw e; - } // Even if history-service is enabled, treat all exceptions still the same // except the following if (e.getClass() != ApplicationNotFoundException.class && e.getClass() != ContainerNotFoundException.class) { throw e; } + if (timelineV2ServiceEnabled) { + try { + return ahsV2Client.getContainerReport(containerId); + } catch (Exception ex) { + LOG.warn("Failed to fetch container report from " + + "ATS v2", ex); + } + } + if (!historyServiceEnabled) { + // Just throw it as usual if historyService is not enabled. + throw e; + } return historyClient.getContainerReport(containerId); } } @@ -802,71 +853,88 @@ public class YarnClientImpl extends YarnClient { GetContainersResponse response = rmClient.getContainers(request); containersForAttempt.addAll(response.getContainerList()); } catch (YarnException e) { - if (e.getClass() != ApplicationNotFoundException.class - || !historyServiceEnabled) { - // If Application is not in RM and history service is enabled then we - // need to check with history service else throw exception. + // Even if history-service is enabled, treat all exceptions still the same + // except the following + if (e.getClass() != ApplicationNotFoundException.class) { + throw e; + } + if (!historyServiceEnabled && !timelineV2ServiceEnabled) { + // if both history server and ATSv2 are not enabled throw exception. throw e; } appNotFoundInRM = true; } - - if (historyServiceEnabled) { - // Check with AHS even if found in RM because to capture info of finished - // containers also - List<ContainerReport> containersListFromAHS = null; - try { - containersListFromAHS = - historyClient.getContainers(applicationAttemptId); - } catch (IOException e) { - // History service access might be enabled but system metrics publisher - // is disabled hence app not found exception is possible - if (appNotFoundInRM) { - // app not found in bothM and RM then propagate the exception. - throw e; - } + // Check with AHS even if found in RM because to capture info of finished + // containers also + List<ContainerReport> containersListFromAHS = null; + try { + containersListFromAHS = + getContainerReportFromHistory(applicationAttemptId); + } catch (IOException e) { + if (appNotFoundInRM) { + throw e; + } + } + if (null != containersListFromAHS && containersListFromAHS.size() > 0) { + // remove duplicates + Set<ContainerId> containerIdsToBeKeptFromAHS = + new HashSet<ContainerId>(); + Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator(); + while (tmpItr.hasNext()) { + containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId()); } - if (null != containersListFromAHS && containersListFromAHS.size() > 0) { - // remove duplicates - - Set<ContainerId> containerIdsToBeKeptFromAHS = - new HashSet<ContainerId>(); - Iterator<ContainerReport> tmpItr = containersListFromAHS.iterator(); - while (tmpItr.hasNext()) { - containerIdsToBeKeptFromAHS.add(tmpItr.next().getContainerId()); - } - - Iterator<ContainerReport> rmContainers = - containersForAttempt.iterator(); - while (rmContainers.hasNext()) { - ContainerReport tmp = rmContainers.next(); - containerIdsToBeKeptFromAHS.remove(tmp.getContainerId()); - // Remove containers from AHS as container from RM will have latest - // information - } + Iterator<ContainerReport> rmContainers = + containersForAttempt.iterator(); + while (rmContainers.hasNext()) { + ContainerReport tmp = rmContainers.next(); + containerIdsToBeKeptFromAHS.remove(tmp.getContainerId()); + // Remove containers from AHS as container from RM will have latest + // information + } - if (containerIdsToBeKeptFromAHS.size() > 0 - && containersListFromAHS.size() != containerIdsToBeKeptFromAHS - .size()) { - Iterator<ContainerReport> containersFromHS = - containersListFromAHS.iterator(); - while (containersFromHS.hasNext()) { - ContainerReport containerReport = containersFromHS.next(); - if (containerIdsToBeKeptFromAHS.contains(containerReport - .getContainerId())) { - containersForAttempt.add(containerReport); - } + if (containerIdsToBeKeptFromAHS.size() > 0 + && containersListFromAHS.size() != containerIdsToBeKeptFromAHS + .size()) { + Iterator<ContainerReport> containersFromHS = + containersListFromAHS.iterator(); + while (containersFromHS.hasNext()) { + ContainerReport containerReport = containersFromHS.next(); + if (containerIdsToBeKeptFromAHS.contains(containerReport + .getContainerId())) { + containersForAttempt.add(containerReport); } - } else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS - .size()) { - containersForAttempt.addAll(containersListFromAHS); } + } else if (containersListFromAHS.size() == containerIdsToBeKeptFromAHS + .size()) { + containersForAttempt.addAll(containersListFromAHS); } } return containersForAttempt; } + private List<ContainerReport> getContainerReportFromHistory( + ApplicationAttemptId applicationAttemptId) + throws IOException, YarnException { + List<ContainerReport> containersListFromAHS = null; + if (timelineV2ServiceEnabled) { + try { + containersListFromAHS = ahsV2Client.getContainers(applicationAttemptId); + } catch (Exception e) { + LOG.warn("Got an error while fetching container report from ATSv2", e); + if (historyServiceEnabled) { + containersListFromAHS = historyClient.getContainers( + applicationAttemptId); + } else { + throw e; + } + } + } else if (historyServiceEnabled) { + containersListFromAHS = historyClient.getContainers(applicationAttemptId); + } + return containersListFromAHS; + } + @Override public void moveApplicationAcrossQueues(ApplicationId appId, String queue) throws YarnException, IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java index cfd4c79..a1550a5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java @@ -96,6 +96,7 @@ public class LogsCLI extends Configured implements Tool { private static final String CONTAINER_ID_OPTION = "containerId"; private static final String APPLICATION_ID_OPTION = "applicationId"; + private static final String CLUSTER_ID_OPTION = "clusterId"; private static final String NODE_ADDRESS_OPTION = "nodeAddress"; private static final String APP_OWNER_OPTION = "appOwner"; private static final String AM_CONTAINER_OPTION = "am"; @@ -134,7 +135,6 @@ public class LogsCLI extends Configured implements Tool { @Override public int run(String[] args) throws Exception { try { - yarnClient = createYarnClient(); webServiceClient = new Client(new URLConnectionClientHandler( new HttpURLConnectionFactory() { @Override @@ -171,6 +171,7 @@ public class LogsCLI extends Configured implements Tool { } CommandLineParser parser = new GnuParser(); String appIdStr = null; + String clusterIdStr = null; String containerIdStr = null; String nodeAddress = null; String appOwner = null; @@ -207,6 +208,10 @@ public class LogsCLI extends Configured implements Tool { return -1; } } + if (commandLine.hasOption(CLUSTER_ID_OPTION)) { + clusterIdStr = commandLine.getOptionValue(CLUSTER_ID_OPTION); + getConf().set(YarnConfiguration.RM_CLUSTER_ID, clusterIdStr); + } if (commandLine.hasOption(PER_CONTAINER_LOG_FILES_OPTION)) { logFiles = commandLine.getOptionValues(PER_CONTAINER_LOG_FILES_OPTION); } @@ -303,6 +308,8 @@ public class LogsCLI extends Configured implements Tool { LogCLIHelpers logCliHelper = new LogCLIHelpers(); logCliHelper.setConf(getConf()); + yarnClient = createYarnClient(); + YarnApplicationState appState = YarnApplicationState.NEW; ApplicationReport appReport = null; try { @@ -824,6 +831,8 @@ public class LogsCLI extends Configured implements Tool { + "By default, it will print all available logs." + " Work with -log_files to get only specific logs. If specified, the" + " applicationId can be omitted"); + opts.addOption(CLUSTER_ID_OPTION, true, "ClusterId. " + + "By default, it will take default cluster id from the RM"); opts.addOption(NODE_ADDRESS_OPTION, true, "NodeAddress in the format " + "nodename:port"); opts.addOption(APP_OWNER_OPTION, true, @@ -892,6 +901,7 @@ public class LogsCLI extends Configured implements Tool { + "and fetch all logs."); opts.getOption(APPLICATION_ID_OPTION).setArgName("Application ID"); opts.getOption(CONTAINER_ID_OPTION).setArgName("Container ID"); + opts.getOption(CLUSTER_ID_OPTION).setArgName("Cluster ID"); opts.getOption(NODE_ADDRESS_OPTION).setArgName("Node Address"); opts.getOption(APP_OWNER_OPTION).setArgName("Application Owner"); opts.getOption(AM_CONTAINER_OPTION).setArgName("AM Containers"); @@ -913,6 +923,7 @@ public class LogsCLI extends Configured implements Tool { Options printOpts = new Options(); printOpts.addOption(commandOpts.getOption(HELP_CMD)); printOpts.addOption(commandOpts.getOption(CONTAINER_ID_OPTION)); + printOpts.addOption(commandOpts.getOption(CLUSTER_ID_OPTION)); printOpts.addOption(commandOpts.getOption(NODE_ADDRESS_OPTION)); printOpts.addOption(commandOpts.getOption(APP_OWNER_OPTION)); printOpts.addOption(commandOpts.getOption(AM_CONTAINER_OPTION)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java new file mode 100644 index 0000000..99473a3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSv2ClientImpl.java @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.client.api.impl; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; +import org.apache.hadoop.yarn.client.api.TimelineReaderClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * This class is to test class {@link AHSv2ClientImpl). + */ +public class TestAHSv2ClientImpl { + + private AHSv2ClientImpl client; + private TimelineReaderClient spyTimelineReaderClient; + @Before + public void setup() { + Configuration conf = new YarnConfiguration(); + conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true); + conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f); + client = new AHSv2ClientImpl(); + spyTimelineReaderClient = mock(TimelineReaderClient.class); + client.setReaderClient(spyTimelineReaderClient); + } + + @Test + public void testGetContainerReport() throws IOException, YarnException { + final ApplicationId appId = ApplicationId.newInstance(0, 1); + final ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + final ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1); + when(spyTimelineReaderClient.getContainerEntity(containerId, "ALL", null)) + .thenReturn(createContainerEntity(containerId)); + ContainerReport report = client.getContainerReport(containerId); + Assert.assertEquals(report.getContainerId(), containerId); + Assert.assertEquals(report.getAssignedNode().getHost(), "test host"); + Assert.assertEquals(report.getAssignedNode().getPort(), 100); + Assert.assertEquals(report.getAllocatedResource().getVirtualCores(), 8); + } + + @Test + public void testGetAppAttemptReport() throws IOException, YarnException { + final ApplicationId appId = ApplicationId.newInstance(0, 1); + final ApplicationAttemptId appAttemptId = + ApplicationAttemptId.newInstance(appId, 1); + when(spyTimelineReaderClient.getApplicationAttemptEntity(appAttemptId, + "ALL", null)) + .thenReturn(createAppAttemptTimelineEntity(appAttemptId)); + ApplicationAttemptReport report = + client.getApplicationAttemptReport(appAttemptId); + Assert.assertEquals(report.getApplicationAttemptId(), appAttemptId); + Assert.assertEquals(report.getFinishTime(), Integer.MAX_VALUE + 2L); + Assert.assertEquals(report.getOriginalTrackingUrl(), + "test original tracking url"); + } + + @Test + public void testGetAppReport() throws IOException, YarnException { + final ApplicationId appId = ApplicationId.newInstance(0, 1); + when(spyTimelineReaderClient.getApplicationEntity(appId, "ALL", null)) + .thenReturn(createApplicationTimelineEntity(appId, false, false)); + ApplicationReport report = client.getApplicationReport(appId); + Assert.assertEquals(report.getApplicationId(), appId); + Assert.assertEquals(report.getAppNodeLabelExpression(), "test_node_label"); + Assert.assertTrue(report.getApplicationTags().contains("Test_APP_TAGS_1")); + Assert.assertEquals(report.getYarnApplicationState(), + YarnApplicationState.FINISHED); + } + + private static TimelineEntity createApplicationTimelineEntity( + ApplicationId appId, boolean emptyACLs, + boolean wrongAppId) { + TimelineEntity entity = new TimelineEntity(); + entity.setType(ApplicationMetricsConstants.ENTITY_TYPE); + if (wrongAppId) { + entity.setId("wrong_app_id"); + } else { + entity.setId(appId.toString()); + } + + Map<String, Object> entityInfo = new HashMap<String, Object>(); + entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app"); + entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO, + "test app type"); + entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "user1"); + entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, + "test queue"); + entityInfo.put( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO, "false"); + entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO, + Priority.newInstance(0)); + entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO, + Integer.MAX_VALUE + 1L); + entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345); + + entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456); + entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789); + + if (emptyACLs) { + entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, ""); + } else { + entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, + "user2"); + } + + Set<String> appTags = new HashSet<String>(); + appTags.add("Test_APP_TAGS_1"); + appTags.add("Test_APP_TAGS_2"); + entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags); + entity.setInfo(entityInfo); + + Map<String, String> configs = new HashMap<>(); + configs.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION, + "test_node_label"); + entity.setConfigs(configs); + + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE); + tEvent.setTimestamp(Integer.MAX_VALUE + 1L + appId.getId()); + entity.addEvent(tEvent); + + // send a YARN_APPLICATION_STATE_UPDATED event + // after YARN_APPLICATION_FINISHED + // The final YarnApplicationState should not be changed + tEvent = new TimelineEvent(); + tEvent.setId( + ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE); + tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId()); + Map<String, Object> eventInfo = new HashMap<>(); + eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, + YarnApplicationState.KILLED); + tEvent.setInfo(eventInfo); + entity.addEvent(tEvent); + + return entity; + } + + private static TimelineEntity createAppAttemptTimelineEntity( + ApplicationAttemptId appAttemptId) { + TimelineEntity entity = new TimelineEntity(); + entity.setType(AppAttemptMetricsConstants.ENTITY_TYPE); + entity.setId(appAttemptId.toString()); + + Map<String, Object> entityInfo = new HashMap<String, Object>(); + entityInfo.put(AppAttemptMetricsConstants.TRACKING_URL_INFO, + "test tracking url"); + entityInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO, + "test original tracking url"); + entityInfo.put(AppAttemptMetricsConstants.HOST_INFO, "test host"); + entityInfo.put(AppAttemptMetricsConstants.RPC_PORT_INFO, 100); + entityInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO, + ContainerId.newContainerId(appAttemptId, 1)); + entity.setInfo(entityInfo); + + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setId(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE); + tEvent.setTimestamp(Integer.MAX_VALUE + 1L); + entity.addEvent(tEvent); + + tEvent = new TimelineEvent(); + tEvent.setId(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE); + tEvent.setTimestamp(Integer.MAX_VALUE + 2L); + entity.addEvent(tEvent); + + return entity; + } + + private static TimelineEntity createContainerEntity(ContainerId containerId) { + TimelineEntity entity = new TimelineEntity(); + entity.setType(ContainerMetricsConstants.ENTITY_TYPE); + entity.setId(containerId.toString()); + Map<String, Object> entityInfo = new HashMap<String, Object>(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, 1024); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, 8); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, + "test host"); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, 100); + entityInfo + .put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, -1); + entityInfo.put(ContainerMetricsConstants + .ALLOCATED_HOST_HTTP_ADDRESS_INFO, "http://test:1234";); + entityInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO, + "test diagnostics info"); + entityInfo.put(ContainerMetricsConstants.EXIT_STATUS_INFO, -1); + entityInfo.put(ContainerMetricsConstants.STATE_INFO, + ContainerState.COMPLETE.toString()); + entity.setInfo(entityInfo); + + TimelineEvent tEvent = new TimelineEvent(); + tEvent.setId(ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE); + tEvent.setTimestamp(123456); + entity.addEvent(tEvent); + + return entity; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java index d8440b8..5366769 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java @@ -246,6 +246,9 @@ public class TestLogsCLI { pw.println(" --client_max_retries to"); pw.println(" create a retry client. The"); pw.println(" default value is 1000."); + pw.println(" -clusterId <Cluster ID> ClusterId. By default, it"); + pw.println(" will take default cluster id"); + pw.println(" from the RM"); pw.println(" -containerId <Container ID> ContainerId. By default, it"); pw.println(" will print all available"); pw.println(" logs. Work with -log_files"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java new file mode 100644 index 0000000..2a62a79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.metrics; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Private +@Unstable +public class AppAttemptMetricsConstants { + + public static final String ENTITY_TYPE = + "YARN_APPLICATION_ATTEMPT"; + + public static final String REGISTERED_EVENT_TYPE = + "YARN_APPLICATION_ATTEMPT_REGISTERED"; + + public static final String FINISHED_EVENT_TYPE = + "YARN_APPLICATION_ATTEMPT_FINISHED"; + + public static final String PARENT_PRIMARY_FILTER = + "YARN_APPLICATION_ATTEMPT_PARENT"; + + public static final String TRACKING_URL_INFO = + "YARN_APPLICATION_ATTEMPT_TRACKING_URL"; + + public static final String ORIGINAL_TRACKING_URL_INFO = + "YARN_APPLICATION_ATTEMPT_ORIGINAL_TRACKING_URL"; + + public static final String HOST_INFO = + "YARN_APPLICATION_ATTEMPT_HOST"; + + public static final String RPC_PORT_INFO = + "YARN_APPLICATION_ATTEMPT_RPC_PORT"; + + public static final String MASTER_CONTAINER_INFO = + "YARN_APPLICATION_ATTEMPT_MASTER_CONTAINER"; + + public static final String DIAGNOSTICS_INFO = + "YARN_APPLICATION_ATTEMPT_DIAGNOSTICS_INFO"; + + public static final String FINAL_STATUS_INFO = + "YARN_APPLICATION_ATTEMPT_FINAL_STATUS"; + + public static final String STATE_INFO = + "YARN_APPLICATION_ATTEMPT_STATE"; + + public static final String MASTER_NODE_ADDRESS = + "YARN_APPLICATION_ATTEMPT_MASTER_NODE_ADDRESS"; + + public static final String MASTER_NODE_ID = + "YARN_APPLICATION_ATTEMPT_MASTER_NODE_ID"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java new file mode 100644 index 0000000..93da565 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.metrics; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Private +@Unstable +public class ApplicationMetricsConstants { + + public static final String ENTITY_TYPE = + "YARN_APPLICATION"; + + public static final String CREATED_EVENT_TYPE = + "YARN_APPLICATION_CREATED"; + + public static final String FINISHED_EVENT_TYPE = + "YARN_APPLICATION_FINISHED"; + + public static final String ACLS_UPDATED_EVENT_TYPE = + "YARN_APPLICATION_ACLS_UPDATED"; + + public static final String UPDATED_EVENT_TYPE = + "YARN_APPLICATION_UPDATED"; + + public static final String STATE_UPDATED_EVENT_TYPE = + "YARN_APPLICATION_STATE_UPDATED"; + + public static final String NAME_ENTITY_INFO = + "YARN_APPLICATION_NAME"; + + public static final String TYPE_ENTITY_INFO = + "YARN_APPLICATION_TYPE"; + + public static final String USER_ENTITY_INFO = + "YARN_APPLICATION_USER"; + + public static final String QUEUE_ENTITY_INFO = + "YARN_APPLICATION_QUEUE"; + + public static final String SUBMITTED_TIME_ENTITY_INFO = + "YARN_APPLICATION_SUBMITTED_TIME"; + + public static final String APP_VIEW_ACLS_ENTITY_INFO = + "YARN_APPLICATION_VIEW_ACLS"; + + public static final String DIAGNOSTICS_INFO_EVENT_INFO = + "YARN_APPLICATION_DIAGNOSTICS_INFO"; + + public static final String FINAL_STATUS_EVENT_INFO = + "YARN_APPLICATION_FINAL_STATUS"; + + public static final String STATE_EVENT_INFO = + "YARN_APPLICATION_STATE"; + + public static final String APP_CPU_METRICS = + "YARN_APPLICATION_CPU"; + + public static final String APP_MEM_METRICS = + "YARN_APPLICATION_MEMORY"; + + public static final String APP_RESOURCE_PREEMPTED_CPU = + "YARN_APPLICATION_RESOURCE_PREEMPTED_CPU"; + + public static final String APP_RESOURCE_PREEMPTED_MEM = + "YARN_APPLICATION_RESOURCE_PREEMPTED_MEMORY"; + + public static final String APP_NON_AM_CONTAINER_PREEMPTED = + "YARN_APPLICATION_NON_AM_CONTAINER_PREEMPTED"; + + public static final String APP_AM_CONTAINER_PREEMPTED = + "YARN_APPLICATION_AM_CONTAINER_PREEMPTED"; + + public static final String APP_CPU_PREEMPT_METRICS = + "YARN_APPLICATION_CPU_PREEMPT_METRIC"; + + public static final String APP_MEM_PREEMPT_METRICS = + "YARN_APPLICATION_MEM_PREEMPT_METRIC"; + + public static final String LATEST_APP_ATTEMPT_EVENT_INFO = + "YARN_APPLICATION_LATEST_APP_ATTEMPT"; + + public static final String YARN_APP_CALLER_CONTEXT = + "YARN_APPLICATION_CALLER_CONTEXT"; + + public static final String YARN_APP_CALLER_SIGNATURE = + "YARN_APPLICATION_CALLER_SIGNATURE"; + + public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS"; + + public static final String UNMANAGED_APPLICATION_ENTITY_INFO = + "YARN_APPLICATION_UNMANAGED_APPLICATION"; + + public static final String APPLICATION_PRIORITY_INFO = + "YARN_APPLICATION_PRIORITY"; + + public static final String APP_NODE_LABEL_EXPRESSION = + "YARN_APP_NODE_LABEL_EXPRESSION"; + + public static final String AM_NODE_LABEL_EXPRESSION = + "YARN_AM_NODE_LABEL_EXPRESSION"; + + public static final String AM_CONTAINER_LAUNCH_COMMAND = + "YARN_AM_CONTAINER_LAUNCH_COMMAND"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java new file mode 100644 index 0000000..9cf2b0a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.metrics; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; + +@Private +@Unstable +public class ContainerMetricsConstants { + + public static final String ENTITY_TYPE = "YARN_CONTAINER"; + + // Event of this type will be emitted by NM. + public static final String CREATED_EVENT_TYPE = "YARN_CONTAINER_CREATED"; + + // Event of this type will be emitted by RM. + public static final String CREATED_IN_RM_EVENT_TYPE = + "YARN_RM_CONTAINER_CREATED"; + + // Event of this type will be emitted by NM. + public static final String FINISHED_EVENT_TYPE = "YARN_CONTAINER_FINISHED"; + + // Event of this type will be emitted by RM. + public static final String FINISHED_IN_RM_EVENT_TYPE = + "YARN_RM_CONTAINER_FINISHED"; + + public static final String CONTAINER_FINISHED_TIME = + "YARN_CONTAINER_FINISHED_TIME"; + + public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT"; + + public static final String ALLOCATED_MEMORY_INFO = + "YARN_CONTAINER_ALLOCATED_MEMORY"; + + public static final String ALLOCATED_VCORE_INFO = + "YARN_CONTAINER_ALLOCATED_VCORE"; + + public static final String ALLOCATED_HOST_INFO = + "YARN_CONTAINER_ALLOCATED_HOST"; + + public static final String ALLOCATED_PORT_INFO = + "YARN_CONTAINER_ALLOCATED_PORT"; + + public static final String ALLOCATED_PRIORITY_INFO = + "YARN_CONTAINER_ALLOCATED_PRIORITY"; + + public static final String DIAGNOSTICS_INFO = + "YARN_CONTAINER_DIAGNOSTICS_INFO"; + + public static final String EXIT_STATUS_INFO = + "YARN_CONTAINER_EXIT_STATUS"; + + public static final String STATE_INFO = + "YARN_CONTAINER_STATE"; + + public static final String ALLOCATED_HOST_HTTP_ADDRESS_INFO = + "YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS"; + + // Event of this type will be emitted by NM. + public static final String LOCALIZATION_START_EVENT_TYPE = + "YARN_NM_CONTAINER_LOCALIZATION_STARTED"; + + // Event of this type will be emitted by NM. + public static final String LOCALIZATION_FINISHED_EVENT_TYPE = + "YARN_NM_CONTAINER_LOCALIZATION_FINISHED"; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java new file mode 100644 index 0000000..b7ef6fb --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** Yarn Common Metrics package. **/ +@InterfaceAudience.Private +package org.apache.hadoop.yarn.server.metrics; +import org.apache.hadoop.classification.InterfaceAudience; + http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java new file mode 100644 index 0000000..7e382a4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java @@ -0,0 +1,449 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.util.timeline; + +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; +import org.apache.hadoop.yarn.api.records.ContainerExitStatus; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerReport; +import org.apache.hadoop.yarn.api.records.ContainerState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent; +import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric; +import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants; +import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.NavigableSet; +import java.util.Set; + +/** + * Utility class to generate reports from timeline entities. + */ +public final class TimelineEntityV2Converter { + private TimelineEntityV2Converter() { + } + + public static ContainerReport convertToContainerReport( + TimelineEntity entity) { + int allocatedMem = 0; + int allocatedVcore = 0; + String allocatedHost = null; + int allocatedPort = -1; + int allocatedPriority = 0; + long createdTime = 0; + long finishedTime = 0; + String diagnosticsInfo = null; + int exitStatus = ContainerExitStatus.INVALID; + ContainerState state = null; + String nodeHttpAddress = null; + Map<String, Object> entityInfo = entity.getInfo(); + if (entityInfo != null) { + if (entityInfo + .containsKey(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO)) { + allocatedMem = (Integer) entityInfo.get( + ContainerMetricsConstants.ALLOCATED_MEMORY_INFO); + } + if (entityInfo + .containsKey(ContainerMetricsConstants.ALLOCATED_VCORE_INFO)) { + allocatedVcore = (Integer) entityInfo.get( + ContainerMetricsConstants.ALLOCATED_VCORE_INFO); + } + if (entityInfo + .containsKey(ContainerMetricsConstants.ALLOCATED_HOST_INFO)) { + allocatedHost = + entityInfo + .get(ContainerMetricsConstants.ALLOCATED_HOST_INFO) + .toString(); + } + if (entityInfo + .containsKey(ContainerMetricsConstants.ALLOCATED_PORT_INFO)) { + allocatedPort = (Integer) entityInfo.get( + ContainerMetricsConstants.ALLOCATED_PORT_INFO); + } + if (entityInfo + .containsKey(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO)) { + allocatedPriority = Integer.parseInt(entityInfo.get( + ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO).toString()); + } + if (entityInfo.containsKey( + ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO)) { + nodeHttpAddress = + (String) entityInfo.get( + ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO); + } + if (entityInfo.containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO)) { + diagnosticsInfo = + entityInfo.get( + ContainerMetricsConstants.DIAGNOSTICS_INFO) + .toString(); + } + if (entityInfo.containsKey(ContainerMetricsConstants.EXIT_STATUS_INFO)) { + exitStatus = (Integer) entityInfo.get( + ContainerMetricsConstants.EXIT_STATUS_INFO); + } + if (entityInfo.containsKey(ContainerMetricsConstants.STATE_INFO)) { + state = + ContainerState.valueOf(entityInfo.get( + ContainerMetricsConstants.STATE_INFO).toString()); + } + } + NavigableSet<TimelineEvent> events = entity.getEvents(); + if (events != null) { + for (TimelineEvent event : events) { + if (event.getId().equals( + ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE)) { + createdTime = event.getTimestamp(); + } else if (event.getId().equals( + ContainerMetricsConstants.FINISHED_IN_RM_EVENT_TYPE)) { + finishedTime = event.getTimestamp(); + } + } + } + String logUrl = null; + NodeId allocatedNode = null; + if (allocatedHost != null) { + allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort); + } + return ContainerReport.newInstance( + ContainerId.fromString(entity.getId()), + Resource.newInstance(allocatedMem, allocatedVcore), allocatedNode, + Priority.newInstance(allocatedPriority), + createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state, + nodeHttpAddress); + } + + public static ApplicationAttemptReport convertToApplicationAttemptReport( + TimelineEntity entity) { + String host = null; + int rpcPort = -1; + ContainerId amContainerId = null; + String trackingUrl = null; + String originalTrackingUrl = null; + String diagnosticsInfo = null; + YarnApplicationAttemptState state = null; + Map<String, Object> entityInfo = entity.getInfo(); + long startTime = 0; + long finishTime = 0; + + if (entityInfo != null) { + if (entityInfo.containsKey(AppAttemptMetricsConstants.HOST_INFO)) { + host = + entityInfo.get(AppAttemptMetricsConstants.HOST_INFO) + .toString(); + } + if (entityInfo + .containsKey(AppAttemptMetricsConstants.RPC_PORT_INFO)) { + rpcPort = (Integer) entityInfo.get( + AppAttemptMetricsConstants.RPC_PORT_INFO); + } + if (entityInfo + .containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO)) { + amContainerId = + ContainerId.fromString(entityInfo.get( + AppAttemptMetricsConstants.MASTER_CONTAINER_INFO) + .toString()); + } + if (entityInfo + .containsKey(AppAttemptMetricsConstants.TRACKING_URL_INFO)) { + trackingUrl = + entityInfo.get( + AppAttemptMetricsConstants.TRACKING_URL_INFO) + .toString(); + } + if (entityInfo + .containsKey( + AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO)) { + originalTrackingUrl = + entityInfo + .get( + AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_INFO) + .toString(); + } + if (entityInfo + .containsKey(AppAttemptMetricsConstants.DIAGNOSTICS_INFO)) { + diagnosticsInfo = + entityInfo.get( + AppAttemptMetricsConstants.DIAGNOSTICS_INFO) + .toString(); + } + if (entityInfo + .containsKey(AppAttemptMetricsConstants.STATE_INFO)) { + state = + YarnApplicationAttemptState.valueOf(entityInfo.get( + AppAttemptMetricsConstants.STATE_INFO) + .toString()); + } + if (entityInfo + .containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_INFO)) { + amContainerId = + ContainerId.fromString(entityInfo.get( + AppAttemptMetricsConstants.MASTER_CONTAINER_INFO) + .toString()); + } + } + NavigableSet<TimelineEvent> events = entity.getEvents(); + if (events != null) { + for (TimelineEvent event : events) { + if (event.getId().equals( + AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) { + startTime = event.getTimestamp(); + } else if (event.getId().equals( + AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) { + finishTime = event.getTimestamp(); + } + } + } + return ApplicationAttemptReport.newInstance( + ApplicationAttemptId.fromString(entity.getId()), + host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo, + state, amContainerId, startTime, finishTime); + } + + public static ApplicationReport convertToApplicationReport( + TimelineEntity entity) { + String user = null; + String queue = null; + String name = null; + String type = null; + boolean unmanagedApplication = false; + long createdTime = 0; + long finishedTime = 0; + float progress = 0.0f; + int applicationPriority = 0; + ApplicationAttemptId latestApplicationAttemptId = null; + String diagnosticsInfo = null; + FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED; + YarnApplicationState state = YarnApplicationState.ACCEPTED; + ApplicationResourceUsageReport appResources = null; + Set<String> appTags = null; + String appNodeLabelExpression = null; + String amNodeLabelExpression = null; + Map<String, Object> entityInfo = entity.getInfo(); + if (entityInfo != null) { + if (entityInfo.containsKey( + ApplicationMetricsConstants.USER_ENTITY_INFO)) { + user = + entityInfo.get(ApplicationMetricsConstants.USER_ENTITY_INFO) + .toString(); + } + if (entityInfo.containsKey( + ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) { + queue = + entityInfo.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO) + .toString(); + } + if (entityInfo.containsKey( + ApplicationMetricsConstants.NAME_ENTITY_INFO)) { + name = + entityInfo.get(ApplicationMetricsConstants.NAME_ENTITY_INFO) + .toString(); + } + if (entityInfo.containsKey( + ApplicationMetricsConstants.TYPE_ENTITY_INFO)) { + type = + entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) + .toString(); + } + if (entityInfo.containsKey( + ApplicationMetricsConstants.TYPE_ENTITY_INFO)) { + type = + entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO) + .toString(); + } + if (entityInfo + .containsKey( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO)) { + unmanagedApplication = + Boolean.parseBoolean(entityInfo.get( + ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO) + .toString()); + } + if (entityInfo + .containsKey(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO)) { + applicationPriority = Integer.parseInt(entityInfo.get( + ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO).toString()); + } + if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) { + appTags = new HashSet<>(); + Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO); + if (obj != null && obj instanceof Collection<?>) { + for(Object o : (Collection<?>)obj) { + if (o != null) { + appTags.add(o.toString()); + } + } + } + } + if (entityInfo + .containsKey( + ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) { + latestApplicationAttemptId = ApplicationAttemptId.fromString( + entityInfo.get( + ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO) + .toString()); + } + if (entityInfo.containsKey( + ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) { + diagnosticsInfo = + entityInfo.get( + ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO) + .toString(); + } + if (entityInfo + .containsKey(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)) { + finalStatus = + FinalApplicationStatus.valueOf(entityInfo.get( + ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO) + .toString()); + } + if (entityInfo + .containsKey(ApplicationMetricsConstants.STATE_EVENT_INFO)) { + state = + YarnApplicationState.valueOf(entityInfo.get( + ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); + } + } + + Map<String, String> configs = entity.getConfigs(); + if (configs + .containsKey(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION)) { + appNodeLabelExpression = configs + .get(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION); + } + if (configs + .containsKey(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION)) { + amNodeLabelExpression = + configs.get(ApplicationMetricsConstants.AM_NODE_LABEL_EXPRESSION); + } + + Set<TimelineMetric> metrics = entity.getMetrics(); + if (metrics != null) { + long vcoreSeconds = 0; + long memorySeconds = 0; + long preemptedVcoreSeconds = 0; + long preemptedMemorySeconds = 0; + + for (TimelineMetric metric : metrics) { + switch (metric.getId()) { + case ApplicationMetricsConstants.APP_CPU_METRICS: + vcoreSeconds = getAverageValue(metric.getValues().values()); + break; + case ApplicationMetricsConstants.APP_MEM_METRICS: + memorySeconds = getAverageValue(metric.getValues().values()); + break; + case ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS: + preemptedVcoreSeconds = getAverageValue(metric.getValues().values()); + break; + case ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS: + preemptedVcoreSeconds = getAverageValue(metric.getValues().values()); + break; + default: + // Should not happen.. + break; + } + } + Map<String, Long> resourceSecondsMap = new HashMap<>(); + Map<String, Long> preemptedResoureSecondsMap = new HashMap<>(); + resourceSecondsMap + .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds); + resourceSecondsMap + .put(ResourceInformation.VCORES.getName(), vcoreSeconds); + preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), + preemptedMemorySeconds); + preemptedResoureSecondsMap + .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds); + + appResources = ApplicationResourceUsageReport + .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0, + preemptedResoureSecondsMap); + } + + NavigableSet<TimelineEvent> events = entity.getEvents(); + long updatedTimeStamp = 0L; + if (events != null) { + for (TimelineEvent event : events) { + if (event.getId().equals( + ApplicationMetricsConstants.CREATED_EVENT_TYPE)) { + createdTime = event.getTimestamp(); + } else if (event.getId().equals( + ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) { + // This type of events are parsed in time-stamp descending order + // which means the previous event could override the information + // from the later same type of event. Hence compare timestamp + // before over writing. + if (event.getTimestamp() > updatedTimeStamp) { + updatedTimeStamp = event.getTimestamp(); + } + } else if (event.getId().equals( + ApplicationMetricsConstants.STATE_UPDATED_EVENT_TYPE)) { + Map<String, Object> eventInfo = event.getInfo(); + if (eventInfo == null) { + continue; + } + if (eventInfo.containsKey( + ApplicationMetricsConstants.STATE_EVENT_INFO)) { + if (state == YarnApplicationState.ACCEPTED) { + state = YarnApplicationState.valueOf(eventInfo.get( + ApplicationMetricsConstants.STATE_EVENT_INFO).toString()); + } + } + } else if (event.getId().equals( + ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) { + progress=1.0F; + state = YarnApplicationState.FINISHED; + finishedTime = event.getTimestamp(); + } + } + } + return ApplicationReport.newInstance( + ApplicationId.fromString(entity.getId()), + latestApplicationAttemptId, user, queue, name, null, -1, null, state, + diagnosticsInfo, null, createdTime, finishedTime, finalStatus, + appResources, null, progress, type, null, appTags, unmanagedApplication, + Priority.newInstance(applicationPriority), appNodeLabelExpression, + amNodeLabelExpression); + } + + private static long getAverageValue(Collection<Number> values) { + if (values == null || values.isEmpty()) { + return 0; + } + long sum = 0; + for (Number value : values) { + sum += value.longValue(); + } + return sum/values.size(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java deleted file mode 100644 index 797aad5..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java +++ /dev/null @@ -1,69 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.metrics; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -@Private -@Unstable -public class AppAttemptMetricsConstants { - - public static final String ENTITY_TYPE = - "YARN_APPLICATION_ATTEMPT"; - - public static final String REGISTERED_EVENT_TYPE = - "YARN_APPLICATION_ATTEMPT_REGISTERED"; - - public static final String FINISHED_EVENT_TYPE = - "YARN_APPLICATION_ATTEMPT_FINISHED"; - - public static final String PARENT_PRIMARY_FILTER = - "YARN_APPLICATION_ATTEMPT_PARENT"; - - public static final String TRACKING_URL_INFO = - "YARN_APPLICATION_ATTEMPT_TRACKING_URL"; - - public static final String ORIGINAL_TRACKING_URL_INFO = - "YARN_APPLICATION_ATTEMPT_ORIGINAL_TRACKING_URL"; - - public static final String HOST_INFO = - "YARN_APPLICATION_ATTEMPT_HOST"; - - public static final String RPC_PORT_INFO = - "YARN_APPLICATION_ATTEMPT_RPC_PORT"; - - public static final String MASTER_CONTAINER_INFO = - "YARN_APPLICATION_ATTEMPT_MASTER_CONTAINER"; - - public static final String DIAGNOSTICS_INFO = - "YARN_APPLICATION_ATTEMPT_DIAGNOSTICS_INFO"; - - public static final String FINAL_STATUS_INFO = - "YARN_APPLICATION_ATTEMPT_FINAL_STATUS"; - - public static final String STATE_INFO = - "YARN_APPLICATION_ATTEMPT_STATE"; - - public static final String MASTER_NODE_ADDRESS = - "YARN_APPLICATION_ATTEMPT_MASTER_NODE_ADDRESS"; - - public static final String MASTER_NODE_ID = - "YARN_APPLICATION_ATTEMPT_MASTER_NODE_ID"; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java deleted file mode 100644 index 4cec409..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java +++ /dev/null @@ -1,122 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.metrics; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -@Private -@Unstable -public class ApplicationMetricsConstants { - - public static final String ENTITY_TYPE = - "YARN_APPLICATION"; - - public static final String CREATED_EVENT_TYPE = - "YARN_APPLICATION_CREATED"; - - public static final String FINISHED_EVENT_TYPE = - "YARN_APPLICATION_FINISHED"; - - public static final String ACLS_UPDATED_EVENT_TYPE = - "YARN_APPLICATION_ACLS_UPDATED"; - - public static final String UPDATED_EVENT_TYPE = - "YARN_APPLICATION_UPDATED"; - - public static final String STATE_UPDATED_EVENT_TYPE = - "YARN_APPLICATION_STATE_UPDATED"; - - public static final String NAME_ENTITY_INFO = - "YARN_APPLICATION_NAME"; - - public static final String TYPE_ENTITY_INFO = - "YARN_APPLICATION_TYPE"; - - public static final String USER_ENTITY_INFO = - "YARN_APPLICATION_USER"; - - public static final String QUEUE_ENTITY_INFO = - "YARN_APPLICATION_QUEUE"; - - public static final String SUBMITTED_TIME_ENTITY_INFO = - "YARN_APPLICATION_SUBMITTED_TIME"; - - public static final String APP_VIEW_ACLS_ENTITY_INFO = - "YARN_APPLICATION_VIEW_ACLS"; - - public static final String DIAGNOSTICS_INFO_EVENT_INFO = - "YARN_APPLICATION_DIAGNOSTICS_INFO"; - - public static final String FINAL_STATUS_EVENT_INFO = - "YARN_APPLICATION_FINAL_STATUS"; - - public static final String STATE_EVENT_INFO = - "YARN_APPLICATION_STATE"; - - public static final String APP_CPU_METRICS = - "YARN_APPLICATION_CPU"; - - public static final String APP_MEM_METRICS = - "YARN_APPLICATION_MEMORY"; - - public static final String APP_RESOURCE_PREEMPTED_CPU = - "YARN_APPLICATION_RESOURCE_PREEMPTED_CPU"; - - public static final String APP_RESOURCE_PREEMPTED_MEM = - "YARN_APPLICATION_RESOURCE_PREEMPTED_MEMORY"; - - public static final String APP_NON_AM_CONTAINER_PREEMPTED = - "YARN_APPLICATION_NON_AM_CONTAINER_PREEMPTED"; - - public static final String APP_AM_CONTAINER_PREEMPTED = - "YARN_APPLICATION_AM_CONTAINER_PREEMPTED"; - - public static final String APP_CPU_PREEMPT_METRICS = - "YARN_APPLICATION_CPU_PREEMPT_METRIC"; - - public static final String APP_MEM_PREEMPT_METRICS = - "YARN_APPLICATION_MEM_PREEMPT_METRIC"; - - public static final String LATEST_APP_ATTEMPT_EVENT_INFO = - "YARN_APPLICATION_LATEST_APP_ATTEMPT"; - - public static final String YARN_APP_CALLER_CONTEXT = - "YARN_APPLICATION_CALLER_CONTEXT"; - - public static final String YARN_APP_CALLER_SIGNATURE = - "YARN_APPLICATION_CALLER_SIGNATURE"; - - public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS"; - - public static final String UNMANAGED_APPLICATION_ENTITY_INFO = - "YARN_APPLICATION_UNMANAGED_APPLICATION"; - - public static final String APPLICATION_PRIORITY_INFO = - "YARN_APPLICATION_PRIORITY"; - - public static final String APP_NODE_LABEL_EXPRESSION = - "YARN_APP_NODE_LABEL_EXPRESSION"; - - public static final String AM_NODE_LABEL_EXPRESSION = - "YARN_AM_NODE_LABEL_EXPRESSION"; - - public static final String AM_CONTAINER_LAUNCH_COMMAND = - "YARN_AM_CONTAINER_LAUNCH_COMMAND"; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee3355be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java deleted file mode 100644 index 9cf2b0a..0000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.metrics; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; - -@Private -@Unstable -public class ContainerMetricsConstants { - - public static final String ENTITY_TYPE = "YARN_CONTAINER"; - - // Event of this type will be emitted by NM. - public static final String CREATED_EVENT_TYPE = "YARN_CONTAINER_CREATED"; - - // Event of this type will be emitted by RM. - public static final String CREATED_IN_RM_EVENT_TYPE = - "YARN_RM_CONTAINER_CREATED"; - - // Event of this type will be emitted by NM. - public static final String FINISHED_EVENT_TYPE = "YARN_CONTAINER_FINISHED"; - - // Event of this type will be emitted by RM. - public static final String FINISHED_IN_RM_EVENT_TYPE = - "YARN_RM_CONTAINER_FINISHED"; - - public static final String CONTAINER_FINISHED_TIME = - "YARN_CONTAINER_FINISHED_TIME"; - - public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT"; - - public static final String ALLOCATED_MEMORY_INFO = - "YARN_CONTAINER_ALLOCATED_MEMORY"; - - public static final String ALLOCATED_VCORE_INFO = - "YARN_CONTAINER_ALLOCATED_VCORE"; - - public static final String ALLOCATED_HOST_INFO = - "YARN_CONTAINER_ALLOCATED_HOST"; - - public static final String ALLOCATED_PORT_INFO = - "YARN_CONTAINER_ALLOCATED_PORT"; - - public static final String ALLOCATED_PRIORITY_INFO = - "YARN_CONTAINER_ALLOCATED_PRIORITY"; - - public static final String DIAGNOSTICS_INFO = - "YARN_CONTAINER_DIAGNOSTICS_INFO"; - - public static final String EXIT_STATUS_INFO = - "YARN_CONTAINER_EXIT_STATUS"; - - public static final String STATE_INFO = - "YARN_CONTAINER_STATE"; - - public static final String ALLOCATED_HOST_HTTP_ADDRESS_INFO = - "YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS"; - - // Event of this type will be emitted by NM. - public static final String LOCALIZATION_START_EVENT_TYPE = - "YARN_NM_CONTAINER_LOCALIZATION_STARTED"; - - // Event of this type will be emitted by NM. - public static final String LOCALIZATION_FINISHED_EVENT_TYPE = - "YARN_NM_CONTAINER_LOCALIZATION_FINISHED"; -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
