YARN-2928: YARN Timeline Service v.2: alpha 1 (sjlee) For a list of commits and contributors, see the merge branch history.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b191c6b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b191c6b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b191c6b2 Branch: refs/heads/trunk Commit: b191c6b2021f5c551310b44975b2986dcfb4a405 Parents: 9bdb5be 6cf6ab7 Author: Sangjin Lee <[email protected]> Authored: Sun Jul 10 08:52:47 2016 -0700 Committer: Sangjin Lee <[email protected]> Committed: Sun Jul 10 08:52:47 2016 -0700 ---------------------------------------------------------------------- .../src/site/markdown/Compatibility.md | 1 + .../jobhistory/JobHistoryEventHandler.java | 323 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 18 + .../mapreduce/v2/app/job/impl/JobImpl.java | 2 +- .../v2/app/rm/RMContainerAllocator.java | 10 + .../hadoop/mapreduce/jobhistory/TestEvents.java | 18 +- .../jobhistory/TestJobHistoryEventHandler.java | 38 +- .../v2/app/rm/TestRMContainerAllocator.java | 3 +- .../mapreduce/jobhistory/AMStartedEvent.java | 28 +- .../mapreduce/jobhistory/HistoryEvent.java | 18 + .../mapreduce/jobhistory/JobFinishedEvent.java | 33 + .../jobhistory/JobInfoChangeEvent.java | 21 +- .../mapreduce/jobhistory/JobInitedEvent.java | 25 +- .../jobhistory/JobPriorityChangeEvent.java | 21 +- .../jobhistory/JobQueueChangeEvent.java | 18 + .../jobhistory/JobStatusChangedEvent.java | 21 +- .../mapreduce/jobhistory/JobSubmittedEvent.java | 67 +- .../JobUnsuccessfulCompletionEvent.java | 27 +- .../jobhistory/MapAttemptFinishedEvent.java | 32 +- .../jobhistory/NormalizedResourceEvent.java | 19 + .../jobhistory/ReduceAttemptFinishedEvent.java | 33 +- .../jobhistory/TaskAttemptFinishedEvent.java | 26 + .../jobhistory/TaskAttemptStartedEvent.java | 29 +- .../TaskAttemptUnsuccessfulCompletionEvent.java | 37 +- .../mapreduce/jobhistory/TaskFailedEvent.java | 29 +- .../mapreduce/jobhistory/TaskFinishedEvent.java | 27 +- .../mapreduce/jobhistory/TaskStartedEvent.java | 20 + .../mapreduce/jobhistory/TaskUpdatedEvent.java | 21 +- .../mapreduce/util/JobHistoryEventUtils.java | 83 + .../hadoop-mapreduce-client-jobclient/pom.xml | 6 + .../mapred/TestMRTimelineEventHandling.java | 313 +- .../org/apache/hadoop/mapred/UtilsForTests.java | 47 +- .../apache/hadoop/mapreduce/EntityWriterV2.java | 57 + .../hadoop/mapreduce/JobHistoryFileParser.java | 3 + .../mapreduce/JobHistoryFileReplayMapperV1.java | 14 +- .../mapreduce/JobHistoryFileReplayMapperV2.java | 161 + .../mapreduce/SimpleEntityWriterConstants.java | 43 + .../hadoop/mapreduce/SimpleEntityWriterV1.java | 32 +- .../hadoop/mapreduce/SimpleEntityWriterV2.java | 131 + .../mapreduce/TimelineEntityConverterV1.java | 15 +- .../mapreduce/TimelineEntityConverterV2.java | 206 + .../mapreduce/TimelineServicePerformance.java | 130 +- .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 22 +- .../apache/hadoop/test/MapredTestDriver.java | 32 +- hadoop-project/pom.xml | 135 +- hadoop-project/src/site/site.xml | 2 + hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 + hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 8 +- .../dev-support/findbugs-exclude.xml | 27 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 + .../api/protocolrecords/AllocateResponse.java | 33 + .../api/records/timeline/TimelineEntity.java | 21 +- .../api/records/timeline/TimelineEvent.java | 8 +- .../ApplicationAttemptEntity.java | 41 + .../timelineservice/ApplicationEntity.java | 52 + .../records/timelineservice/ClusterEntity.java | 40 + .../timelineservice/ContainerEntity.java | 41 + .../timelineservice/FlowActivityEntity.java | 191 + .../records/timelineservice/FlowRunEntity.java | 126 + .../HierarchicalTimelineEntity.java | 133 + .../records/timelineservice/QueueEntity.java | 40 + .../timelineservice/TimelineEntities.java | 62 + .../records/timelineservice/TimelineEntity.java | 584 +++ .../timelineservice/TimelineEntityType.java | 101 + .../records/timelineservice/TimelineEvent.java | 133 + .../records/timelineservice/TimelineMetric.java | 289 ++ .../TimelineMetricCalculator.java | 115 + .../TimelineMetricOperation.java | 167 + .../timelineservice/TimelineWriteResponse.java | 167 + .../api/records/timelineservice/UserEntity.java | 40 + .../records/timelineservice/package-info.java | 26 + .../hadoop/yarn/conf/YarnConfiguration.java | 157 +- .../hadoop/yarn/util/TimelineServiceHelper.java | 49 + .../src/main/proto/yarn_service_protos.proto | 1 + .../timelineservice/TestTimelineMetric.java | 100 + .../yarn/conf/TestYarnConfigurationFields.java | 3 + .../pom.xml | 10 + .../distributedshell/ApplicationMaster.java | 177 +- .../applications/distributedshell/Client.java | 45 +- .../distributedshell/TestDistributedShell.java | 298 +- .../TestDistributedShellWithNodeLabels.java | 2 +- .../hadoop/yarn/client/api/AMRMClient.java | 20 +- .../yarn/client/api/async/AMRMClientAsync.java | 19 +- .../api/async/impl/AMRMClientAsyncImpl.java | 17 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 + .../impl/pb/AllocateResponsePBImpl.java | 17 + .../hadoop/yarn/client/api/TimelineClient.java | 87 +- .../client/api/impl/TimelineClientImpl.java | 488 ++- .../yarn/util/timeline/TimelineUtils.java | 49 + .../hadoop/yarn/webapp/util/WebAppUtils.java | 8 +- .../src/main/resources/webapps/timeline/.keep | 0 .../src/main/resources/yarn-default.xml | 87 +- .../hadoop/yarn/TestContainerLaunchRPC.java | 16 +- .../yarn/TestContainerResourceIncreaseRPC.java | 16 +- .../java/org/apache/hadoop/yarn/TestRPC.java | 265 -- .../TestTimelineServiceRecords.java | 312 ++ .../api/impl/TestTimelineClientV2Impl.java | 378 ++ .../yarn/util/TestTimelineServiceHelper.java | 83 + .../hadoop-yarn-server-common/pom.xml | 1 + .../api/CollectorNodemanagerProtocol.java | 73 + .../api/CollectorNodemanagerProtocolPB.java | 34 + ...ollectorNodemanagerProtocolPBClientImpl.java | 114 + ...llectorNodemanagerProtocolPBServiceImpl.java | 82 + .../GetTimelineCollectorContextRequest.java | 37 + .../GetTimelineCollectorContextResponse.java | 51 + .../protocolrecords/NodeHeartbeatRequest.java | 23 + .../protocolrecords/NodeHeartbeatResponse.java | 4 + .../ReportNewCollectorInfoRequest.java | 53 + .../ReportNewCollectorInfoResponse.java | 32 + ...etTimelineCollectorContextRequestPBImpl.java | 132 + ...tTimelineCollectorContextResponsePBImpl.java | 159 + .../impl/pb/NodeHeartbeatRequestPBImpl.java | 62 + .../impl/pb/NodeHeartbeatResponsePBImpl.java | 47 + .../pb/ReportNewCollectorInfoRequestPBImpl.java | 144 + .../ReportNewCollectorInfoResponsePBImpl.java | 76 + .../server/api/records/AppCollectorsMap.java | 46 + .../records/impl/pb/AppCollectorsMapPBImpl.java | 152 + .../metrics/ContainerMetricsConstants.java | 18 + .../proto/collectornodemanager_protocol.proto | 30 + .../yarn_server_common_service_protos.proto | 31 + .../java/org/apache/hadoop/yarn/TestRPC.java | 416 ++ .../hadoop/yarn/TestYarnServerApiClasses.java | 17 + .../hadoop/yarn/server/nodemanager/Context.java | 15 + .../yarn/server/nodemanager/NodeManager.java | 77 +- .../nodemanager/NodeStatusUpdaterImpl.java | 47 +- .../collectormanager/NMCollectorService.java | 145 + .../collectormanager/package-info.java | 28 + .../containermanager/ContainerManagerImpl.java | 113 +- .../application/Application.java | 5 + .../ApplicationContainerFinishedEvent.java | 18 +- .../application/ApplicationImpl.java | 91 +- .../containermanager/container/Container.java | 3 + .../container/ContainerImpl.java | 23 +- .../monitor/ContainersMonitorImpl.java | 32 +- .../timelineservice/NMTimelineEvent.java | 35 + .../timelineservice/NMTimelineEventType.java | 27 + .../timelineservice/NMTimelinePublisher.java | 405 ++ .../timelineservice/package-info.java | 29 + .../yarn/server/nodemanager/TestEventFlow.java | 2 +- .../nodemanager/TestNodeStatusUpdater.java | 10 +- .../amrmproxy/BaseAMRMProxyTest.java | 19 + .../BaseContainerManagerTest.java | 3 +- .../TestContainerManagerRecovery.java | 13 +- .../application/TestApplication.java | 13 +- .../launcher/TestContainerLaunch.java | 21 +- .../TestLocalCacheDirectoryManager.java | 2 +- .../TestResourceLocalizationService.java | 5 +- .../TestAppLogAggregatorImpl.java | 4 +- .../TestNMTimelinePublisher.java | 157 + .../yarn/server/nodemanager/webapp/MockApp.java | 22 + .../nodemanager/webapp/MockContainer.java | 5 + .../webapp/TestContainerLogsPage.java | 15 +- .../nodemanager/webapp/TestNMAppsPage.java | 2 +- .../nodemanager/webapp/TestNMWebServer.java | 8 +- .../nodemanager/webapp/TestNMWebServices.java | 6 +- .../webapp/TestNMWebServicesApps.java | 2 +- .../webapp/TestNMWebServicesContainers.java | 2 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 10 + .../ApplicationMasterService.java | 11 + .../server/resourcemanager/ClientRMService.java | 24 + .../resourcemanager/RMActiveServiceContext.java | 20 +- .../server/resourcemanager/RMAppManager.java | 28 +- .../yarn/server/resourcemanager/RMContext.java | 6 + .../server/resourcemanager/RMContextImpl.java | 19 +- .../server/resourcemanager/ResourceManager.java | 63 +- .../resourcemanager/ResourceTrackerService.java | 68 + .../resourcemanager/amlauncher/AMLauncher.java | 58 +- .../metrics/AbstractSystemMetricsPublisher.java | 178 + .../metrics/AppAttemptFinishedEvent.java | 90 - .../metrics/AppAttemptRegisteredEvent.java | 81 - .../metrics/ApplicaitonStateUpdatedEvent.java | 47 - .../metrics/ApplicationACLsUpdatedEvent.java | 45 - .../metrics/ApplicationCreatedEvent.java | 124 - .../metrics/ApplicationFinishedEvent.java | 82 - .../metrics/ApplicationUpdatedEvent.java | 54 - .../metrics/ContainerCreatedEvent.java | 73 - .../metrics/ContainerFinishedEvent.java | 72 - .../metrics/NoOpSystemMetricPublisher.java | 70 + .../metrics/SystemMetricsEvent.java | 33 - .../metrics/SystemMetricsEventType.java | 37 - .../metrics/SystemMetricsPublisher.java | 608 +-- .../metrics/TimelineServiceV1Publisher.java | 386 ++ .../metrics/TimelineServiceV2Publisher.java | 449 +++ .../resourcemanager/metrics/package-info.java | 28 + .../server/resourcemanager/rmapp/RMApp.java | 23 + .../rmapp/RMAppCollectorUpdateEvent.java | 40 + .../resourcemanager/rmapp/RMAppEventType.java | 3 + .../server/resourcemanager/rmapp/RMAppImpl.java | 93 +- .../RMTimelineCollectorManager.java | 104 + .../timelineservice/package-info.java | 28 + .../server/resourcemanager/TestAppManager.java | 5 + .../resourcemanager/TestClientRMService.java | 15 +- .../server/resourcemanager/TestRMRestart.java | 63 + .../TestResourceTrackerService.java | 86 + .../applicationsmanager/MockAsm.java | 12 + .../TestRMAppLogAggregationStatus.java | 6 +- .../metrics/TestSystemMetricsPublisher.java | 6 +- .../TestSystemMetricsPublisherForV2.java | 398 ++ .../server/resourcemanager/rmapp/MockRMApp.java | 14 + .../rmapp/TestRMAppTransitions.java | 4 + .../hadoop-yarn-server-tests/pom.xml | 17 + .../hadoop/yarn/server/MiniYARNCluster.java | 7 +- .../hadoop/yarn/server/TestMiniYarnCluster.java | 2 - .../TestTimelineServiceClientIntegration.java | 180 + .../pom.xml | 401 ++ ...stTimelineReaderWebServicesHBaseStorage.java | 2166 ++++++++++ .../storage/TestHBaseTimelineStorage.java | 3751 ++++++++++++++++++ ...TestPhoenixOfflineAggregationWriterImpl.java | 161 + .../storage/flow/TestFlowDataGenerator.java | 386 ++ .../flow/TestHBaseStorageFlowActivity.java | 483 +++ .../storage/flow/TestHBaseStorageFlowRun.java | 1034 +++++ .../flow/TestHBaseStorageFlowRunCompaction.java | 831 ++++ .../src/test/resources/log4j.properties | 19 + .../hadoop-yarn-server-timelineservice/pom.xml | 203 + .../server/timelineservice/TimelineContext.java | 146 + .../collector/AppLevelTimelineCollector.java | 161 + .../collector/NodeTimelineCollectorManager.java | 223 ++ .../PerNodeTimelineCollectorsAuxService.java | 231 ++ .../collector/TimelineCollector.java | 341 ++ .../collector/TimelineCollectorContext.java | 76 + .../collector/TimelineCollectorManager.java | 254 ++ .../collector/TimelineCollectorWebService.java | 250 ++ .../timelineservice/collector/package-info.java | 29 + .../server/timelineservice/package-info.java | 28 + .../reader/TimelineDataToRetrieve.java | 147 + .../reader/TimelineEntityFilters.java | 242 ++ .../reader/TimelineParseConstants.java | 34 + .../reader/TimelineParseException.java | 36 + .../timelineservice/reader/TimelineParser.java | 37 + .../reader/TimelineParserForCompareExpr.java | 300 ++ .../reader/TimelineParserForDataToRetrieve.java | 95 + .../reader/TimelineParserForEqualityExpr.java | 343 ++ .../reader/TimelineParserForExistFilters.java | 51 + .../reader/TimelineParserForKVFilters.java | 78 + .../reader/TimelineParserForNumericFilters.java | 72 + .../TimelineParserForRelationFilters.java | 71 + .../reader/TimelineReaderContext.java | 98 + .../reader/TimelineReaderManager.java | 179 + .../reader/TimelineReaderServer.java | 178 + .../reader/TimelineReaderUtils.java | 171 + .../reader/TimelineReaderWebServices.java | 2123 ++++++++++ .../reader/TimelineReaderWebServicesUtils.java | 247 ++ .../reader/TimelineUIDConverter.java | 247 ++ .../reader/filter/TimelineCompareFilter.java | 147 + .../reader/filter/TimelineCompareOp.java | 36 + .../reader/filter/TimelineExistsFilter.java | 107 + .../reader/filter/TimelineFilter.java | 68 + .../reader/filter/TimelineFilterList.java | 141 + .../reader/filter/TimelineFilterUtils.java | 290 ++ .../reader/filter/TimelineKeyValueFilter.java | 61 + .../reader/filter/TimelineKeyValuesFilter.java | 126 + .../reader/filter/TimelinePrefixFilter.java | 99 + .../reader/filter/package-info.java | 28 + .../timelineservice/reader/package-info.java | 29 + .../storage/FileSystemTimelineReaderImpl.java | 408 ++ .../storage/FileSystemTimelineWriterImpl.java | 158 + .../storage/HBaseTimelineReaderImpl.java | 88 + .../storage/HBaseTimelineWriterImpl.java | 571 +++ .../storage/OfflineAggregationWriter.java | 67 + .../PhoenixOfflineAggregationWriterImpl.java | 358 ++ .../storage/TimelineAggregationTrack.java | 28 + .../timelineservice/storage/TimelineReader.java | 180 + .../storage/TimelineSchemaCreator.java | 272 ++ .../timelineservice/storage/TimelineWriter.java | 87 + .../storage/application/ApplicationColumn.java | 156 + .../application/ApplicationColumnFamily.java | 65 + .../application/ApplicationColumnPrefix.java | 288 ++ .../storage/application/ApplicationRowKey.java | 206 + .../application/ApplicationRowKeyPrefix.java | 69 + .../storage/application/ApplicationTable.java | 161 + .../storage/application/package-info.java | 28 + .../storage/apptoflow/AppToFlowColumn.java | 148 + .../apptoflow/AppToFlowColumnFamily.java | 51 + .../storage/apptoflow/AppToFlowRowKey.java | 143 + .../storage/apptoflow/AppToFlowTable.java | 113 + .../storage/apptoflow/package-info.java | 28 + .../storage/common/AppIdKeyConverter.java | 95 + .../storage/common/BaseTable.java | 140 + .../common/BufferedMutatorDelegator.java | 73 + .../timelineservice/storage/common/Column.java | 80 + .../storage/common/ColumnFamily.java | 34 + .../storage/common/ColumnHelper.java | 388 ++ .../storage/common/ColumnPrefix.java | 145 + .../storage/common/EventColumnName.java | 63 + .../common/EventColumnNameConverter.java | 99 + .../storage/common/GenericConverter.java | 48 + .../storage/common/KeyConverter.java | 41 + .../storage/common/LongConverter.java | 94 + .../storage/common/LongKeyConverter.java | 68 + .../storage/common/NumericValueConverter.java | 39 + .../storage/common/OfflineAggregationInfo.java | 115 + .../timelineservice/storage/common/Range.java | 62 + .../storage/common/RowKeyPrefix.java | 42 + .../storage/common/Separator.java | 575 +++ .../storage/common/StringKeyConverter.java | 54 + .../common/TimelineEntityFiltersType.java | 71 + .../common/TimelineHBaseSchemaConstants.java | 71 + .../storage/common/TimelineStorageUtils.java | 586 +++ .../storage/common/TimestampGenerator.java | 116 + .../storage/common/TypedBufferedMutator.java | 28 + .../storage/common/ValueConverter.java | 47 + .../storage/common/package-info.java | 28 + .../storage/entity/EntityColumn.java | 160 + .../storage/entity/EntityColumnFamily.java | 65 + .../storage/entity/EntityColumnPrefix.java | 300 ++ .../storage/entity/EntityRowKey.java | 225 ++ .../storage/entity/EntityRowKeyPrefix.java | 74 + .../storage/entity/EntityTable.java | 161 + .../storage/entity/package-info.java | 28 + .../flow/AggregationCompactionDimension.java | 63 + .../storage/flow/AggregationOperation.java | 94 + .../timelineservice/storage/flow/Attribute.java | 39 + .../storage/flow/FlowActivityColumnFamily.java | 55 + .../storage/flow/FlowActivityColumnPrefix.java | 277 ++ .../storage/flow/FlowActivityRowKey.java | 196 + .../storage/flow/FlowActivityRowKeyPrefix.java | 60 + .../storage/flow/FlowActivityTable.java | 108 + .../storage/flow/FlowRunColumn.java | 182 + .../storage/flow/FlowRunColumnFamily.java | 54 + .../storage/flow/FlowRunColumnPrefix.java | 268 ++ .../storage/flow/FlowRunCoprocessor.java | 304 ++ .../storage/flow/FlowRunRowKey.java | 190 + .../storage/flow/FlowRunRowKeyPrefix.java | 54 + .../storage/flow/FlowRunTable.java | 141 + .../storage/flow/FlowScanner.java | 727 ++++ .../storage/flow/FlowScannerOperation.java | 46 + .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../storage/reader/ApplicationEntityReader.java | 481 +++ .../reader/FlowActivityEntityReader.java | 163 + .../storage/reader/FlowRunEntityReader.java | 269 ++ .../storage/reader/GenericEntityReader.java | 648 +++ .../storage/reader/TimelineEntityReader.java | 496 +++ .../reader/TimelineEntityReaderFactory.java | 89 + .../storage/reader/package-info.java | 28 + .../TestAppLevelTimelineCollector.java | 23 + .../TestNMTimelineCollectorManager.java | 170 + ...TestPerNodeTimelineCollectorsAuxService.java | 203 + .../collector/TestTimelineCollector.java | 127 + .../reader/TestTimelineReaderServer.java | 57 + .../reader/TestTimelineReaderUtils.java | 55 + .../reader/TestTimelineReaderWebServices.java | 562 +++ .../TestTimelineReaderWebServicesUtils.java | 923 +++++ .../reader/TestTimelineUIDConverter.java | 97 + .../TestFileSystemTimelineReaderImpl.java | 804 ++++ .../TestFileSystemTimelineWriterImpl.java | 129 + .../storage/common/TestKeyConverters.java | 130 + .../storage/common/TestRowKeys.java | 246 ++ .../storage/common/TestSeparator.java | 215 + .../src/test/resources/log4j.properties | 19 + .../hadoop-yarn/hadoop-yarn-server/pom.xml | 2 + .../src/site/markdown/TimelineServer.md | 2 +- .../src/site/markdown/TimelineServiceV2.md | 1199 ++++++ .../site/resources/images/flow_hierarchy.png | Bin 0 -> 42345 bytes .../src/site/resources/images/timeline_v2.jpg | Bin 0 -> 45112 bytes 355 files changed, 47599 insertions(+), 2060 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
