Merge branch 'trunk' into HADOOP-13345 Only conflicts were from HADOOP-13989, which was straightforward (remove source jar option).
Conflicts: hadoop-client-modules/hadoop-client-minicluster/pom.xml hadoop-client-modules/hadoop-client-runtime/pom.xml Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ba10b4a7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ba10b4a7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ba10b4a7 Branch: refs/heads/HADOOP-13345 Commit: ba10b4a77d824b308a3e49003368bb1ec1c58f95 Parents: b22c0d6 11e44bd Author: Aaron Fabbri <fab...@apache.org> Authored: Mon Jan 30 15:31:34 2017 -0800 Committer: Aaron Fabbri <fab...@apache.org> Committed: Mon Jan 30 15:31:34 2017 -0800 ---------------------------------------------------------------------- BUILDING.txt | 2 + dev-support/bin/create-release | 5 +- hadoop-assemblies/pom.xml | 4 +- .../main/resources/assemblies/hadoop-dist.xml | 14 - .../assemblies/hadoop-mapreduce-dist.xml | 14 - .../resources/assemblies/hadoop-yarn-dist.xml | 14 - hadoop-build-tools/pom.xml | 2 +- hadoop-client-modules/hadoop-client-api/pom.xml | 4 +- .../hadoop-client-check-invariants/pom.xml | 4 +- .../hadoop-client-check-test-invariants/pom.xml | 4 +- .../hadoop-client-integration-tests/pom.xml | 4 +- .../hadoop-client-minicluster/pom.xml | 5 +- .../hadoop-client-runtime/pom.xml | 5 +- hadoop-client-modules/hadoop-client/pom.xml | 4 +- hadoop-client-modules/pom.xml | 2 +- .../hadoop-cloud-storage/pom.xml | 4 +- hadoop-cloud-storage-project/pom.xml | 4 +- .../hadoop-annotations/pom.xml | 4 +- .../hadoop-auth-examples/pom.xml | 4 +- hadoop-common-project/hadoop-auth/pom.xml | 4 +- hadoop-common-project/hadoop-common/pom.xml | 9 +- .../hadoop-common/src/main/bin/hadoop | 13 +- .../src/main/bin/hadoop-functions.sh | 215 ++++- .../hadoop-common/src/main/bin/start-all.sh | 23 +- .../hadoop-common/src/main/bin/stop-all.sh | 19 +- .../crypto/key/kms/KMSClientProvider.java | 20 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 2 +- .../java/org/apache/hadoop/fs/GlobPattern.java | 2 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 47 +- .../org/apache/hadoop/http/HttpServer2.java | 4 +- .../hadoop/metrics2/lib/RollingAverages.java | 57 +- .../AuthenticationFilterInitializer.java | 9 +- .../AuthenticationWithProxyUserFilter.java | 119 +++ .../java/org/apache/hadoop/security/Groups.java | 11 +- .../hadoop/security/UserGroupInformation.java | 65 +- .../security/ssl/FileBasedKeyStoresFactory.java | 2 +- .../apache/hadoop/security/ssl/SSLFactory.java | 28 +- .../AbstractDelegationTokenSecretManager.java | 6 +- .../main/java/org/apache/hadoop/util/Shell.java | 52 +- .../src/site/markdown/CredentialProviderAPI.md | 2 + .../hadoop-common/src/site/markdown/Metrics.md | 53 +- .../src/site/markdown/UnixShellGuide.md | 9 + .../3.0.0-alpha2/CHANGES.3.0.0-alpha2.md | 927 +++++++++++++++++++ .../3.0.0-alpha2/RELEASENOTES.3.0.0-alpha2.md | 618 +++++++++++++ .../hadoop/fs/TestDelegateToFsCheckPath.java | 35 +- .../org/apache/hadoop/fs/TestGlobPattern.java | 8 +- .../org/apache/hadoop/http/TestHttpServer.java | 2 +- .../hadoop/http/TestHttpServerWithSpengo.java | 239 +++++ .../apache/hadoop/http/TestSSLHttpServer.java | 2 +- .../metrics2/lib/TestRollingAverages.java | 13 +- .../security/TestAuthenticationFilter.java | 13 +- .../TestAuthenticationWithProxyUserFilter.java | 78 ++ .../security/TestFixKerberosTicketOrder.java | 158 ++++ .../hadoop/security/TestRaceWhenRelogin.java | 162 ++++ .../java/org/apache/hadoop/util/TestShell.java | 4 +- .../scripts/hadoop-functions_test_helper.bash | 6 +- .../src/test/scripts/hadoop_abs.bats | 65 ++ .../test/scripts/hadoop_get_verify_uservar.bats | 21 + .../test/scripts/hadoop_privilege_check.bats | 26 + hadoop-common-project/hadoop-kms/pom.xml | 20 +- .../crypto/key/kms/server/KMSWebServer.java | 21 +- .../hadoop-kms/src/site/markdown/index.md.vm | 3 + .../hadoop/crypto/key/kms/server/MiniKMS.java | 11 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 6 +- hadoop-common-project/hadoop-minikdc/pom.xml | 4 +- hadoop-common-project/hadoop-nfs/pom.xml | 4 +- .../org/apache/hadoop/oncrpc/RpcProgram.java | 30 +- .../apache/hadoop/oncrpc/SimpleUdpClient.java | 11 +- hadoop-common-project/pom.xml | 4 +- hadoop-dist/pom.xml | 14 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 +- .../apache/hadoop/hdfs/protocol/BlockType.java | 57 ++ .../hadoop/hdfs/protocol/DatanodeInfo.java | 2 +- .../hadoop/hdfs/protocol/LocatedBlock.java | 4 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 20 + .../hdfs/server/protocol/SlowPeerReports.java | 107 +++ .../src/main/proto/hdfs.proto | 10 +- .../hadoop/hdfs/protocol/TestBlockType.java | 61 ++ hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +- .../hadoop-hdfs-native-client/pom.xml | 4 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 +- .../hadoop/hdfs/nfs/conf/NfsConfigKeys.java | 6 + .../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 4 +- .../nfs/nfs3/PrivilegedNfsGatewayStarter.java | 17 +- .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 4 +- .../org/apache/hadoop/hdfs/nfs/TestMountd.java | 13 +- .../jdiff/Apache_Hadoop_HDFS_3.0.0-alpha2.xml | 326 +++++++ hadoop-hdfs-project/hadoop-hdfs/pom.xml | 12 +- .../hadoop-hdfs/src/main/bin/hdfs | 11 + .../hadoop-hdfs/src/main/bin/start-dfs.sh | 50 +- .../src/main/bin/start-secure-dns.sh | 23 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 34 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 22 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 44 +- .../DatanodeProtocolClientSideTranslatorPB.java | 9 +- .../DatanodeProtocolServerSideTranslatorPB.java | 3 +- .../NamenodeProtocolServerSideTranslatorPB.java | 16 + .../NamenodeProtocolTranslatorPB.java | 15 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 44 + .../hadoop/hdfs/qjournal/client/QuorumCall.java | 26 +- .../hadoop/hdfs/server/balancer/Balancer.java | 3 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 3 +- .../server/blockmanagement/BlockIdManager.java | 16 +- .../hdfs/server/blockmanagement/BlockInfo.java | 7 +- .../blockmanagement/BlockInfoContiguous.java | 10 + .../blockmanagement/BlockInfoStriped.java | 27 +- .../server/blockmanagement/BlockManager.java | 70 +- .../blockmanagement/BlockPlacementPolicies.java | 13 +- .../BlockUnderConstructionFeature.java | 9 +- .../server/blockmanagement/DatanodeManager.java | 42 +- .../server/blockmanagement/SlowPeerTracker.java | 273 ++++++ .../hadoop/hdfs/server/common/Storage.java | 4 +- .../hdfs/server/datanode/BPServiceActor.java | 38 +- .../server/datanode/BlockPoolSliceStorage.java | 2 +- .../hdfs/server/datanode/BlockReceiver.java | 17 +- .../hadoop/hdfs/server/datanode/DNConf.java | 11 + .../hadoop/hdfs/server/datanode/DataNode.java | 8 +- .../hdfs/server/datanode/DataStorage.java | 2 +- .../hdfs/server/datanode/DataXceiver.java | 4 +- .../server/datanode/ProfilingFileIoEvents.java | 23 +- .../checker/StorageLocationChecker.java | 11 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 9 +- .../datanode/metrics/DataNodePeerMetrics.java | 79 +- .../datanode/metrics/SlowNodeDetector.java | 194 ++++ .../server/datanode/web/DatanodeHttpServer.java | 16 +- .../web/PortUnificationServerHandler.java | 99 -- .../datanode/web/SimpleHttpProxyHandler.java | 18 +- .../hdfs/server/datanode/web/URLDispatcher.java | 10 +- .../datanode/web/dtp/DtpHttp2FrameListener.java | 52 -- .../datanode/web/dtp/DtpHttp2Handler.java | 34 - .../datanode/web/webhdfs/ExceptionHandler.java | 29 +- .../server/datanode/web/webhdfs/HdfsWriter.java | 10 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 52 +- .../hdfs/server/namenode/Checkpointer.java | 4 +- .../hdfs/server/namenode/EditsDoubleBuffer.java | 34 + .../hdfs/server/namenode/FSDirTruncateOp.java | 4 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 53 +- .../hadoop/hdfs/server/namenode/FSImage.java | 4 +- .../hdfs/server/namenode/FSImageFormat.java | 4 +- .../server/namenode/FSImageFormatPBINode.java | 15 +- .../hdfs/server/namenode/FSNamesystem.java | 31 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 48 +- .../server/namenode/INodeFileAttributes.java | 13 +- .../hadoop/hdfs/server/namenode/NameNode.java | 6 + .../hdfs/server/namenode/NameNodeRpcServer.java | 16 +- .../server/namenode/NameNodeStatusMXBean.java | 6 + .../hdfs/server/namenode/NamenodeFsck.java | 49 +- .../hdfs/server/namenode/SecondaryNameNode.java | 6 +- .../snapshot/FSImageFormatPBSnapshot.java | 2 +- .../hdfs/server/protocol/DatanodeProtocol.java | 8 +- .../hdfs/server/protocol/NamenodeProtocol.java | 8 + .../org/apache/hadoop/hdfs/tools/DFSck.java | 6 +- .../offlineImageViewer/FSImageHandler.java | 41 +- .../OfflineImageReconstructor.java | 16 +- .../offlineImageViewer/PBImageXmlWriter.java | 9 +- .../src/main/proto/DatanodeProtocol.proto | 20 + .../src/main/proto/NamenodeProtocol.proto | 16 + .../hadoop-hdfs/src/main/proto/fsimage.proto | 4 +- .../src/main/resources/hdfs-default.xml | 18 +- .../src/main/webapps/hdfs/dfshealth.html | 3 + .../src/main/webapps/hdfs/dfshealth.js | 65 ++ .../src/main/webapps/static/d3-v4.1.1.min.js | 8 + .../src/main/webapps/static/hadoop.css | 9 + .../src/site/markdown/ArchivalStorage.md | 14 +- .../site/markdown/CentralizedCacheManagement.md | 25 +- .../src/site/markdown/ExtendedAttributes.md | 8 +- .../hadoop-hdfs/src/site/markdown/Federation.md | 15 +- .../src/site/markdown/HDFSCommands.md | 44 +- .../src/site/markdown/HDFSDiskbalancer.md | 6 +- .../src/site/markdown/HDFSErasureCoding.md | 10 +- .../markdown/HDFSHighAvailabilityWithNFS.md | 30 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 31 +- .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 36 +- .../src/site/markdown/HdfsEditsViewer.md | 5 +- .../src/site/markdown/HdfsImageViewer.md | 11 +- .../src/site/markdown/HdfsMultihoming.md | 9 +- .../src/site/markdown/HdfsNfsGateway.md | 9 +- .../src/site/markdown/HdfsPermissionsGuide.md | 15 +- .../src/site/markdown/HdfsQuotaAdminGuide.md | 10 +- .../src/site/markdown/HdfsRollingUpgrade.md | 18 +- .../src/site/markdown/HdfsSnapshots.md | 16 +- .../src/site/markdown/HdfsUserGuide.md | 23 +- .../hadoop-hdfs/src/site/markdown/LibHdfs.md | 8 +- .../src/site/markdown/MemoryStorage.md | 14 +- .../src/site/markdown/SLGUserGuide.md | 7 +- .../src/site/markdown/ShortCircuitLocalReads.md | 7 +- .../src/site/markdown/TransparentEncryption.md | 27 +- .../hadoop-hdfs/src/site/markdown/ViewFs.md | 15 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 123 +-- .../apache/hadoop/hdfs/AdminStatesBaseTest.java | 151 ++- .../apache/hadoop/hdfs/TestDecommission.java | 43 + .../hadoop/hdfs/TestMaintenanceState.java | 48 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 37 + .../hdfs/qjournal/client/TestQuorumCall.java | 17 + .../server/blockmanagement/TestBlockInfo.java | 20 +- .../blockmanagement/TestBlockInfoStriped.java | 17 + .../blockmanagement/TestHeartbeatHandling.java | 10 + .../TestNameNodePrunesMissingStorages.java | 4 +- .../blockmanagement/TestSlowPeerTracker.java | 226 +++++ .../datanode/InternalDataNodeTestUtils.java | 4 +- .../server/datanode/TestBPOfferService.java | 6 +- .../hdfs/server/datanode/TestBlockRecovery.java | 4 +- .../datanode/TestBpServiceActorScheduler.java | 30 +- .../server/datanode/TestDataNodeLifeline.java | 7 +- .../datanode/TestDataNodePeerMetrics.java | 8 +- .../datanode/TestDataNodeVolumeFailure.java | 22 +- .../TestDatanodeProtocolRetryPolicy.java | 4 +- .../server/datanode/TestFsDatasetCache.java | 3 +- .../hdfs/server/datanode/TestStorageReport.java | 4 +- .../checker/TestStorageLocationChecker.java | 4 +- .../TestDataNodeOutlierDetectionViaMetrics.java | 142 +++ .../datanode/metrics/TestSlowNodeDetector.java | 335 +++++++ .../datanode/web/dtp/Http2ResponseHandler.java | 65 -- .../server/datanode/web/dtp/TestDtpHttp2.java | 147 --- .../hdfs/server/namenode/CreateEditsLog.java | 2 +- .../server/namenode/NNThroughputBenchmark.java | 7 +- .../hdfs/server/namenode/NameNodeAdapter.java | 4 +- .../hdfs/server/namenode/TestAuditLogger.java | 73 ++ .../hdfs/server/namenode/TestDeadDatanode.java | 7 +- .../server/namenode/TestEditsDoubleBuffer.java | 54 ++ .../hdfs/server/namenode/TestFSImage.java | 97 +- .../server/namenode/TestFSNamesystemMBean.java | 10 + .../hadoop/hdfs/server/namenode/TestFsck.java | 119 ++- .../hdfs/server/namenode/TestINodeFile.java | 24 +- .../server/namenode/TestNameNodeMXBean.java | 2 +- .../server/namenode/TestStripedINodeFile.java | 4 +- .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 13 +- .../hadoop/tools/TestHdfsConfigFields.java | 6 + hadoop-hdfs-project/pom.xml | 4 +- hadoop-mapreduce-project/bin/mapred | 10 + .../hadoop-mapreduce-client-app/pom.xml | 4 +- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 6 +- .../hadoop-mapreduce-client-common/pom.xml | 4 +- .../hadoop-mapreduce-client-core/pom.xml | 4 +- .../java/org/apache/hadoop/mapred/Task.java | 24 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 2 + .../apache/hadoop/mapreduce/TaskCounter.java | 8 +- .../counters/FrameworkCounterGroup.java | 6 +- .../src/main/resources/mapred-default.xml | 18 + .../hadoop/mapreduce/TaskCounter.properties | 4 + .../org/apache/hadoop/mapred/TestCounters.java | 31 +- .../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 +- .../hadoop-mapreduce-client-hs/pom.xml | 4 +- .../hadoop-mapreduce-client-jobclient/pom.xml | 4 +- .../org/apache/hadoop/mapred/YARNRunner.java | 36 + .../java/org/apache/hadoop/mapred/MRBench.java | 24 +- .../apache/hadoop/mapred/TestJobCounters.java | 188 ++++ .../apache/hadoop/mapred/TestYARNRunner.java | 39 + .../hadoop-mapreduce-client-nativetask/pom.xml | 4 +- .../hadoop-mapreduce-client-shuffle/pom.xml | 4 +- .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- .../hadoop-mapreduce-client/pom.xml | 4 +- .../hadoop-mapreduce-examples/pom.xml | 4 +- hadoop-mapreduce-project/pom.xml | 4 +- hadoop-maven-plugins/pom.xml | 2 +- hadoop-minicluster/pom.xml | 4 +- hadoop-project-dist/pom.xml | 4 +- hadoop-project/pom.xml | 25 +- hadoop-project/src/site/markdown/index.md.vm | 40 +- hadoop-project/src/site/site.xml | 1 + hadoop-tools/hadoop-aliyun/pom.xml | 2 +- hadoop-tools/hadoop-archive-logs/pom.xml | 4 +- hadoop-tools/hadoop-archives/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 4 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 4 +- .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 37 +- .../src/site/markdown/index.md | 44 + .../hadoop/fs/adl/TestAzureADTokenProvider.java | 152 +++ hadoop-tools/hadoop-azure/pom.xml | 2 +- hadoop-tools/hadoop-datajoin/pom.xml | 4 +- hadoop-tools/hadoop-distcp/pom.xml | 4 +- .../tools/mapred/RetriableFileCopyCommand.java | 7 +- hadoop-tools/hadoop-extras/pom.xml | 4 +- hadoop-tools/hadoop-gridmix/pom.xml | 4 +- .../gridmix/DummyResourceCalculatorPlugin.java | 4 + hadoop-tools/hadoop-kafka/pom.xml | 4 +- hadoop-tools/hadoop-openstack/pom.xml | 4 +- hadoop-tools/hadoop-pipes/pom.xml | 4 +- hadoop-tools/hadoop-rumen/pom.xml | 4 +- hadoop-tools/hadoop-sls/pom.xml | 4 +- hadoop-tools/hadoop-streaming/pom.xml | 4 +- hadoop-tools/hadoop-tools-dist/pom.xml | 4 +- hadoop-tools/pom.xml | 4 +- .../hadoop-yarn/bin/start-yarn.sh | 25 +- .../hadoop-yarn/bin/stop-yarn.sh | 17 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 11 + .../dev-support/findbugs-exclude.xml | 4 - .../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +- .../api/protocolrecords/AllocateRequest.java | 4 +- .../api/records/ContainerLaunchContext.java | 16 + .../hadoop/yarn/conf/YarnConfiguration.java | 7 +- .../src/main/proto/yarn_protos.proto | 2 + .../src/main/proto/yarn_service_protos.proto | 2 +- .../pom.xml | 4 +- .../pom.xml | 4 +- .../hadoop-yarn-applications/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-client/pom.xml | 4 +- .../yarn/client/api/SharedCacheClient.java | 23 +- .../client/api/impl/SharedCacheClientImpl.java | 31 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 85 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 38 +- .../yarn/client/api/impl/TestAMRMProxy.java | 11 +- .../api/impl/TestSharedCacheClientImpl.java | 37 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 47 +- .../hadoop-yarn/hadoop-yarn-common/pom.xml | 5 +- .../impl/pb/ContainerLaunchContextPBImpl.java | 26 + .../logaggregation/AggregatedLogFormat.java | 2 +- .../yarn/logaggregation/ContainerLogMeta.java | 61 ++ .../yarn/logaggregation/ContainerLogType.java | 31 + .../logaggregation/LogAggregationUtils.java | 64 +- .../yarn/logaggregation/LogCLIHelpers.java | 119 +-- .../yarn/logaggregation/LogToolUtils.java | 117 +++ .../logaggregation/PerContainerLogFileInfo.java | 93 ++ .../main/resources/webapps/sharedcache/.keep | 0 .../src/main/resources/yarn-default.xml | 41 +- .../logaggregation/TestContainerLogsUtils.java | 122 +++ .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 +- .../client/impl/zk/RegistrySecurity.java | 11 +- .../registry/secure/TestSecureRegistry.java | 22 + .../pom.xml | 4 +- .../webapp/AHSWebServices.java | 154 +++ .../webapp/TestAHSWebServices.java | 238 +++-- .../hadoop-yarn-server-common/pom.xml | 4 +- .../hadoop/yarn/server/utils/BuilderUtils.java | 32 + .../server/webapp/dao/ContainerLogsInfo.java | 87 ++ .../hadoop-yarn-server-nodemanager/pom.xml | 4 +- .../AMRMProxyApplicationContextImpl.java | 2 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 5 + .../containermanager/ContainerManagerImpl.java | 7 + .../CGroupsBlkioResourceHandlerImpl.java | 2 +- .../CGroupsCpuResourceHandlerImpl.java | 2 +- .../linux/resources/CGroupsHandler.java | 58 +- .../linux/resources/CGroupsHandlerImpl.java | 171 +++- .../CGroupsMemoryResourceHandlerImpl.java | 2 +- .../TrafficControlBandwidthHandlerImpl.java | 3 +- .../localizer/ContainerLocalizer.java | 53 +- .../recovery/NMLeveldbStateStoreService.java | 19 +- .../recovery/NMStateStoreService.java | 17 + .../nodemanager/webapp/NMWebServices.java | 56 +- .../webapp/dao/ContainerLogsInfo.java | 112 --- .../webapp/dao/NMContainerLogsInfo.java | 79 ++ .../amrmproxy/MockResourceManagerFacade.java | 7 +- .../amrmproxy/TestAMRMProxyService.java | 13 + .../TestCGroupsBlkioResourceHandlerImpl.java | 2 +- .../TestCGroupsCpuResourceHandlerImpl.java | 6 +- .../linux/resources/TestCGroupsHandlerImpl.java | 153 ++- .../TestCGroupsMemoryResourceHandlerImpl.java | 2 +- .../TestTrafficControlBandwidthHandlerImpl.java | 2 +- .../localizer/TestContainerLocalizer.java | 317 +++++-- .../TestNMLeveldbStateStoreService.java | 69 ++ .../nodemanager/webapp/TestNMWebServices.java | 64 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 4 +- .../server/resourcemanager/AdminService.java | 2 +- .../server/resourcemanager/ClientRMService.java | 58 +- .../server/resourcemanager/RMAppManager.java | 84 +- .../server/resourcemanager/ResourceManager.java | 26 +- .../AbstractPreemptableResourceCalculator.java | 14 +- .../capacity/PreemptionCandidatesSelector.java | 3 +- .../ProportionalCapacityPreemptionPolicy.java | 37 + ...QueuePriorityContainerCandidateSelector.java | 510 ++++++++++ .../monitor/capacity/TempQueuePerPartition.java | 53 +- .../monitor/capacity/TempSchedulerNode.java | 120 +++ .../server/resourcemanager/rmapp/RMAppImpl.java | 23 +- .../rmcontainer/RMContainerImpl.java | 7 +- .../scheduler/SchedulerApplicationAttempt.java | 19 +- .../scheduler/SchedulerNode.java | 23 +- .../scheduler/capacity/AbstractCSQueue.java | 10 + .../scheduler/capacity/CSQueue.java | 7 + .../scheduler/capacity/CapacityScheduler.java | 79 +- .../CapacitySchedulerConfiguration.java | 236 ++++- .../capacity/CapacitySchedulerContext.java | 4 - .../capacity/CapacitySchedulerQueueManager.java | 60 +- .../scheduler/capacity/LeafQueue.java | 52 +- .../scheduler/capacity/ParentQueue.java | 74 +- .../capacity/PartitionedQueueComparator.java | 72 -- .../PriorityUtilizationQueueOrderingPolicy.java | 192 ++++ .../capacity/policy/QueueOrderingPolicy.java | 52 ++ .../scheduler/common/fica/FiCaSchedulerApp.java | 96 +- .../fair/AllocationFileLoaderService.java | 26 +- .../scheduler/fair/FSAppAttempt.java | 20 +- .../scheduler/fair/FSParentQueue.java | 15 - .../scheduler/fair/FSPreemptionThread.java | 135 ++- .../resourcemanager/scheduler/fair/FSQueue.java | 27 +- .../scheduler/fair/QueueManager.java | 15 +- .../scheduler/fair/Schedulable.java | 8 +- .../security/DelegationTokenRenewer.java | 74 +- .../security/QueueACLsManager.java | 69 +- .../yarn/server/resourcemanager/MockAM.java | 4 +- .../yarn/server/resourcemanager/MockRM.java | 36 +- .../ParameterizedSchedulerTestBase.java | 1 + .../server/resourcemanager/TestAppManager.java | 41 +- .../resourcemanager/TestClientRMService.java | 258 +++++- .../server/resourcemanager/TestRMRestart.java | 110 ++- .../TestResourceTrackerService.java | 4 + .../TestRMAppLogAggregationStatus.java | 8 +- ...alCapacityPreemptionPolicyMockFramework.java | 80 +- .../TestPreemptionForQueueWithPriorities.java | 361 ++++++++ ...estProportionalCapacityPreemptionPolicy.java | 91 +- ...pacityPreemptionPolicyForNodePartitions.java | 11 +- ...alCapacityPreemptionPolicyMockFramework.java | 13 +- .../rmapp/TestRMAppTransitions.java | 2 +- .../TestSchedulerApplicationAttempt.java | 12 + .../CapacitySchedulerPreemptionTestBase.java | 22 +- .../capacity/TestApplicationLimits.java | 9 - .../TestApplicationLimitsByPartition.java | 3 - .../capacity/TestCapacityScheduler.java | 192 +++- ...TestCapacitySchedulerSurgicalPreemption.java | 572 +++++++++++- .../scheduler/capacity/TestChildQueueOrder.java | 3 - .../capacity/TestContainerAllocation.java | 196 ++++ .../scheduler/capacity/TestLeafQueue.java | 44 +- .../scheduler/capacity/TestParentQueue.java | 3 - .../scheduler/capacity/TestQueueParsing.java | 58 ++ .../scheduler/capacity/TestReservations.java | 2 - ...tPriorityUtilizationQueueOrderingPolicy.java | 222 +++++ .../scheduler/fair/FairSchedulerTestBase.java | 10 +- .../scheduler/fair/FakeSchedulable.java | 5 + .../fair/TestAllocationFileLoaderService.java | 88 ++ .../scheduler/fair/TestFSLeafQueue.java | 1 - .../fair/TestFairSchedulerPreemption.java | 112 ++- .../scheduler/fair/TestSchedulingPolicy.java | 5 + .../policy/TestFairOrderingPolicy.java | 11 +- .../security/TestDelegationTokenRenewer.java | 127 ++- .../pom.xml | 4 +- .../hadoop-yarn-server-tests/pom.xml | 4 +- .../pom.xml | 4 +- .../pom.xml | 16 +- .../pom.xml | 191 ++++ .../reader/filter/TimelineFilterUtils.java | 290 ++++++ .../reader/filter/package-info.java | 28 + .../storage/HBaseTimelineReaderImpl.java | 88 ++ .../storage/HBaseTimelineWriterImpl.java | 566 +++++++++++ .../storage/TimelineSchemaCreator.java | 250 +++++ .../storage/application/ApplicationColumn.java | 156 ++++ .../application/ApplicationColumnFamily.java | 65 ++ .../application/ApplicationColumnPrefix.java | 288 ++++++ .../storage/application/ApplicationRowKey.java | 206 +++++ .../application/ApplicationRowKeyPrefix.java | 69 ++ .../storage/application/ApplicationTable.java | 161 ++++ .../storage/application/package-info.java | 28 + .../storage/apptoflow/AppToFlowColumn.java | 148 +++ .../apptoflow/AppToFlowColumnFamily.java | 51 + .../storage/apptoflow/AppToFlowRowKey.java | 143 +++ .../storage/apptoflow/AppToFlowTable.java | 113 +++ .../storage/apptoflow/package-info.java | 28 + .../storage/common/AppIdKeyConverter.java | 96 ++ .../storage/common/BaseTable.java | 140 +++ .../common/BufferedMutatorDelegator.java | 73 ++ .../timelineservice/storage/common/Column.java | 80 ++ .../storage/common/ColumnFamily.java | 34 + .../storage/common/ColumnHelper.java | 388 ++++++++ .../storage/common/ColumnPrefix.java | 145 +++ .../storage/common/EventColumnName.java | 63 ++ .../common/EventColumnNameConverter.java | 99 ++ .../storage/common/GenericConverter.java | 48 + .../common/HBaseTimelineStorageUtils.java | 243 +++++ .../storage/common/KeyConverter.java | 41 + .../storage/common/LongConverter.java | 94 ++ .../storage/common/LongKeyConverter.java | 68 ++ .../storage/common/NumericValueConverter.java | 39 + .../timelineservice/storage/common/Range.java | 62 ++ .../storage/common/RowKeyPrefix.java | 42 + .../storage/common/Separator.java | 575 ++++++++++++ .../storage/common/StringKeyConverter.java | 54 ++ .../common/TimelineHBaseSchemaConstants.java | 71 ++ .../storage/common/TimestampGenerator.java | 116 +++ .../storage/common/TypedBufferedMutator.java | 28 + .../storage/common/ValueConverter.java | 47 + .../storage/common/package-info.java | 28 + .../storage/entity/EntityColumn.java | 160 ++++ .../storage/entity/EntityColumnFamily.java | 65 ++ .../storage/entity/EntityColumnPrefix.java | 300 ++++++ .../storage/entity/EntityRowKey.java | 225 +++++ .../storage/entity/EntityRowKeyPrefix.java | 74 ++ .../storage/entity/EntityTable.java | 161 ++++ .../storage/entity/package-info.java | 28 + .../flow/AggregationCompactionDimension.java | 63 ++ .../storage/flow/AggregationOperation.java | 94 ++ .../timelineservice/storage/flow/Attribute.java | 39 + .../storage/flow/FlowActivityColumnFamily.java | 55 ++ .../storage/flow/FlowActivityColumnPrefix.java | 277 ++++++ .../storage/flow/FlowActivityRowKey.java | 196 ++++ .../storage/flow/FlowActivityRowKeyPrefix.java | 60 ++ .../storage/flow/FlowActivityTable.java | 108 +++ .../storage/flow/FlowRunColumn.java | 182 ++++ .../storage/flow/FlowRunColumnFamily.java | 54 ++ .../storage/flow/FlowRunColumnPrefix.java | 268 ++++++ .../storage/flow/FlowRunCoprocessor.java | 304 ++++++ .../storage/flow/FlowRunRowKey.java | 190 ++++ .../storage/flow/FlowRunRowKeyPrefix.java | 54 ++ .../storage/flow/FlowRunTable.java | 141 +++ .../storage/flow/FlowScanner.java | 728 +++++++++++++++ .../storage/flow/FlowScannerOperation.java | 46 + .../storage/flow/package-info.java | 29 + .../timelineservice/storage/package-info.java | 28 + .../storage/reader/ApplicationEntityReader.java | 481 ++++++++++ .../reader/FlowActivityEntityReader.java | 163 ++++ .../storage/reader/FlowRunEntityReader.java | 269 ++++++ .../storage/reader/GenericEntityReader.java | 648 +++++++++++++ .../storage/reader/TimelineEntityReader.java | 496 ++++++++++ .../reader/TimelineEntityReaderFactory.java | 89 ++ .../storage/reader/package-info.java | 28 + .../storage/common/TestKeyConverters.java | 130 +++ .../storage/common/TestRowKeys.java | 246 +++++ .../storage/common/TestSeparator.java | 215 +++++ .../hadoop-yarn-server-timelineservice/pom.xml | 70 +- .../reader/filter/TimelineFilterUtils.java | 290 ------ .../storage/HBaseTimelineReaderImpl.java | 88 -- .../storage/HBaseTimelineWriterImpl.java | 566 ----------- .../storage/TimelineSchemaCreator.java | 250 ----- .../storage/application/ApplicationColumn.java | 156 ---- .../application/ApplicationColumnFamily.java | 65 -- .../application/ApplicationColumnPrefix.java | 288 ------ .../storage/application/ApplicationRowKey.java | 206 ----- .../application/ApplicationRowKeyPrefix.java | 69 -- .../storage/application/ApplicationTable.java | 161 ---- .../storage/application/package-info.java | 28 - .../storage/apptoflow/AppToFlowColumn.java | 148 --- .../apptoflow/AppToFlowColumnFamily.java | 51 - .../storage/apptoflow/AppToFlowRowKey.java | 143 --- .../storage/apptoflow/AppToFlowTable.java | 113 --- .../storage/apptoflow/package-info.java | 28 - .../storage/common/AppIdKeyConverter.java | 96 -- .../storage/common/BaseTable.java | 140 --- .../common/BufferedMutatorDelegator.java | 73 -- .../timelineservice/storage/common/Column.java | 80 -- .../storage/common/ColumnFamily.java | 34 - .../storage/common/ColumnHelper.java | 388 -------- .../storage/common/ColumnPrefix.java | 145 --- .../storage/common/EventColumnName.java | 63 -- .../common/EventColumnNameConverter.java | 99 -- .../storage/common/GenericConverter.java | 48 - .../common/HBaseTimelineStorageUtils.java | 243 ----- .../storage/common/KeyConverter.java | 41 - .../storage/common/LongConverter.java | 94 -- .../storage/common/LongKeyConverter.java | 68 -- .../storage/common/NumericValueConverter.java | 39 - .../timelineservice/storage/common/Range.java | 62 -- .../storage/common/RowKeyPrefix.java | 42 - .../storage/common/Separator.java | 575 ------------ .../storage/common/StringKeyConverter.java | 54 -- .../common/TimelineHBaseSchemaConstants.java | 71 -- .../storage/common/TimestampGenerator.java | 116 --- .../storage/common/TypedBufferedMutator.java | 28 - .../storage/common/ValueConverter.java | 47 - .../storage/common/package-info.java | 28 - .../storage/entity/EntityColumn.java | 160 ---- .../storage/entity/EntityColumnFamily.java | 65 -- .../storage/entity/EntityColumnPrefix.java | 300 ------ .../storage/entity/EntityRowKey.java | 225 ----- .../storage/entity/EntityRowKeyPrefix.java | 74 -- .../storage/entity/EntityTable.java | 161 ---- .../storage/entity/package-info.java | 28 - .../flow/AggregationCompactionDimension.java | 63 -- .../storage/flow/AggregationOperation.java | 94 -- .../timelineservice/storage/flow/Attribute.java | 39 - .../storage/flow/FlowActivityColumnFamily.java | 55 -- .../storage/flow/FlowActivityColumnPrefix.java | 277 ------ .../storage/flow/FlowActivityRowKey.java | 196 ---- .../storage/flow/FlowActivityRowKeyPrefix.java | 60 -- .../storage/flow/FlowActivityTable.java | 108 --- .../storage/flow/FlowRunColumn.java | 182 ---- .../storage/flow/FlowRunColumnFamily.java | 54 -- .../storage/flow/FlowRunColumnPrefix.java | 268 ------ .../storage/flow/FlowRunCoprocessor.java | 304 ------ .../storage/flow/FlowRunRowKey.java | 190 ---- .../storage/flow/FlowRunRowKeyPrefix.java | 54 -- .../storage/flow/FlowRunTable.java | 141 --- .../storage/flow/FlowScanner.java | 728 --------------- .../storage/flow/FlowScannerOperation.java | 46 - .../storage/flow/package-info.java | 29 - .../storage/reader/ApplicationEntityReader.java | 481 ---------- .../reader/FlowActivityEntityReader.java | 163 ---- .../storage/reader/FlowRunEntityReader.java | 269 ------ .../storage/reader/GenericEntityReader.java | 648 ------------- .../storage/reader/TimelineEntityReader.java | 496 ---------- .../reader/TimelineEntityReaderFactory.java | 89 -- .../storage/reader/package-info.java | 28 - .../storage/common/TestKeyConverters.java | 130 --- .../storage/common/TestRowKeys.java | 246 ----- .../storage/common/TestSeparator.java | 215 ----- .../hadoop-yarn-server-web-proxy/pom.xml | 4 +- .../hadoop-yarn/hadoop-yarn-server/pom.xml | 5 +- .../hadoop-yarn/hadoop-yarn-site/pom.xml | 4 +- .../src/site/markdown/FairScheduler.md | 2 + .../src/site/markdown/NodeLabel.md | 8 + .../src/site/markdown/NodeManagerCgroups.md | 2 +- .../site/markdown/OpportunisticContainers.md | 6 +- .../src/site/markdown/TimelineServiceV2.md | 2 +- .../src/site/markdown/YarnCommands.md | 2 + .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 4 +- hadoop-yarn-project/hadoop-yarn/pom.xml | 4 +- hadoop-yarn-project/pom.xml | 8 +- pom.xml | 3 +- 593 files changed, 26422 insertions(+), 15030 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba10b4a7/hadoop-project/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ba10b4a7/hadoop-tools/hadoop-aws/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
