Cleanup changes during rebase with trunk (Vrushali C)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d943038 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d943038 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d943038 Branch: refs/heads/YARN-2928 Commit: 6d943038f61211594d017f631710e6710ec71372 Parents: 1a22774 Author: Vrushali <vrush...@twitter.com> Authored: Tue Jun 21 16:38:20 2016 -0700 Committer: Sangjin Lee <sj...@apache.org> Committed: Sun Jul 10 08:46:04 2016 -0700 ---------------------------------------------------------------------- hadoop-project/pom.xml | 5 ----- hadoop-yarn-project/hadoop-yarn/bin/yarn | 11 +++++----- .../nodemanager/NodeStatusUpdaterImpl.java | 1 - .../containermanager/ContainerManagerImpl.java | 6 ++---- .../application/ApplicationImpl.java | 22 +++++++++----------- .../metrics/TimelineServiceV1Publisher.java | 12 +++++++++++ .../metrics/TimelineServiceV2Publisher.java | 16 +++++++++++++- .../pom.xml | 4 ++-- 8 files changed, 46 insertions(+), 31 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-project/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml index bb46de0..e6858a2 100644 --- a/hadoop-project/pom.xml +++ b/hadoop-project/pom.xml @@ -1133,11 +1133,6 @@ <scope>test</scope> <optional>true</optional> <exclusions> - <!-- exclude jdk.tools (1.7) as we're not managing it --> - <exclusion> - <groupId>jdk.tools</groupId> - <artifactId>jdk.tools</artifactId> - </exclusion> <exclusion> <groupId>org.jruby</groupId> <artifactId>jruby-complete</artifactId> http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn index a01f495..dbbaed3 100755 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -48,11 +48,10 @@ function hadoop_usage hadoop_add_subcommand "rmadmin" "admin tools" hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools" hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon" - hadoop_add_subcommand "timelineserver" "run the timeline server" hadoop_add_subcommand "timelinereader" "run the timeline reader server" + hadoop_add_subcommand "timelineserver" "run the timeline server" hadoop_add_subcommand "top" "view cluster information" hadoop_add_subcommand "version" "print the version" - hadoop_generate_usage "${MYNAME}" true hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true } @@ -173,6 +172,10 @@ function yarncmd_case hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}" ;; + timelinereader) + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' + ;; timelineserver) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer' @@ -184,10 +187,6 @@ function yarncmd_case HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}" fi ;; - timelinereader) - supportdaemonization="true" - CLASS='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer' - ;; version) HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS" http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java index 6e0e760..b37b38e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java @@ -81,7 +81,6 @@ import org.apache.hadoop.yarn.server.api.records.NodeAction; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 663a2e1..89e81ca 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -154,7 +154,6 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils; import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; -import org.apache.hadoop.yarn.util.resource.Resources; public class ContainerManagerImpl extends CompositeService implements ContainerManager { @@ -351,9 +350,8 @@ public class ContainerManagerImpl extends CompositeService implements LOG.info("Recovering application " + appId); //TODO: Recover flow and flow run ID - ApplicationImpl app = new ApplicationImpl( - dispatcher, p.getUser(), null, appId, creds, context, - p.getAppLogAggregationInitedTime()); + ApplicationImpl app = new ApplicationImpl(dispatcher, p.getUser(), appId, + creds, context, p.getAppLogAggregationInitedTime()); context.getApplications().put(appId, app); app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java index 5a5d30c..531693e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java @@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.LogAggregationContext; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; -import org.apache.hadoop.yarn.client.api.TimelineClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.proto.YarnProtos; @@ -118,10 +117,19 @@ public class ApplicationImpl implements Application { Context context, long recoveredLogInitedTime) { this.dispatcher = dispatcher; this.user = user; - this.flowContext = flowContext; this.appId = appId; this.credentials = credentials; this.aclsManager = context.getApplicationACLsManager(); + Configuration conf = context.getConf(); + if (YarnConfiguration.timelineServiceV2Enabled(conf)) { + if (flowContext == null) { + throw new IllegalArgumentException("flow context cannot be null"); + } + this.flowContext = flowContext; + if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { + context.getNMTimelinePublisher().createTimelineClient(appId); + } + } this.context = context; this.appStateStore = context.getNMStateStore(); ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); @@ -136,16 +144,6 @@ public class ApplicationImpl implements Application { Credentials credentials, Context context) { this(dispatcher, user, flowContext, appId, credentials, context, -1); - Configuration conf = context.getConf(); - if (YarnConfiguration.timelineServiceV2Enabled(conf)) { - if (flowContext == null) { - throw new IllegalArgumentException("flow context cannot be null"); - } - this.flowContext = flowContext; - if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) { - context.getNMTimelinePublisher().createTimelineClient(appId); - } - } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java index ed8cf06..61e7d65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java @@ -181,6 +181,8 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); } + @SuppressWarnings("unchecked") + @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { TimelineEntity entity = createApplicationEntity(app.getApplicationId()); TimelineEvent tEvent = new TimelineEvent(); @@ -248,6 +250,10 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { app.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); + if (appAttempt.getMasterContainer() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + appAttempt.getMasterContainer().getId().toString()); + } tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); @@ -301,6 +307,12 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher { container.getContainerExitStatus()); eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, container.getContainerState().toString()); + Map<String, Object> entityInfo = new HashMap<String, Object>(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, + container.getAllocatedNode().getHost()); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, + container.getAllocatedNode().getPort()); + entity.setOtherInfo(entityInfo); tEvent.setEventInfo(eventInfo); entity.addEvent(tEvent); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java index 955570f..a248199 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java @@ -196,10 +196,14 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { @Override public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) { ApplicationEntity entity = createApplicationEntity(app.getApplicationId()); + TimelineEvent tEvent = new TimelineEvent(); Map<String, Object> entityInfo = new HashMap<String, Object>(); entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, - appViewACLs); + (appViewACLs == null) ? "" : appViewACLs); entity.setInfo(entityInfo); + tEvent.setId(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE); + tEvent.setTimestamp(updatedTime); + entity.addEvent(tEvent); getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent( SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId())); @@ -285,6 +289,10 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { app.getFinalApplicationStatus().toString()); eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, RMServerUtils .createApplicationAttemptState(appAttemtpState).toString()); + if (appAttempt.getMasterContainer() != null) { + eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, + appAttempt.getMasterContainer().getId().toString()); + } tEvent.setInfo(eventInfo); entity.addEvent(tEvent); @@ -353,6 +361,12 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher { container.getContainerExitStatus()); eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, container.getContainerState().toString()); + Map<String, Object> entityInfo = new HashMap<String, Object>(); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, + container.getAllocatedNode().getHost()); + entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, + container.getAllocatedNode().getPort()); + entity.setInfo(entityInfo); tEvent.setInfo(eventInfo); entity.addEvent(tEvent); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d943038/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml index a8e5195..5641572 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase-tests/pom.xml @@ -22,12 +22,12 @@ <parent> <artifactId>hadoop-yarn-server</artifactId> <groupId>org.apache.hadoop</groupId> - <version>3.0.0-SNAPSHOT</version> + <version>3.0.0-alpha1-SNAPSHOT</version> </parent> <modelVersion>4.0.0</modelVersion> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-timelineservice-hbase-tests</artifactId> - <version>3.0.0-SNAPSHOT</version> + <version>3.0.0-alpha1-SNAPSHOT</version> <name>Apache Hadoop YARN Timeline Service HBase tests</name> <properties> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org