Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae6c0e83 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae6c0e83 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae6c0e83 Branch: refs/heads/HDFS-7240 Commit: ae6c0e839bf36d9b3dc85898a00c340452f8f999 Parents: eee7269 27b131e Author: Anu Engineer <[email protected]> Authored: Wed Apr 13 23:13:11 2016 -0700 Committer: Anu Engineer <[email protected]> Committed: Wed Apr 13 23:13:11 2016 -0700 ---------------------------------------------------------------------- .../server/AuthenticationFilter.java | 63 +- .../util/ZKSignerSecretProvider.java | 29 +- .../src/site/markdown/Configuration.md | 50 +- .../apache/hadoop/crypto/CryptoInputStream.java | 6 +- .../apache/hadoop/fs/CachingGetSpaceUsed.java | 168 +++ .../apache/hadoop/fs/ChecksumFileSystem.java | 20 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 24 +- .../src/main/java/org/apache/hadoop/fs/DU.java | 240 +--- .../org/apache/hadoop/fs/FSDataInputStream.java | 10 +- .../apache/hadoop/fs/FSExceptionMessages.java | 6 + .../org/apache/hadoop/fs/FSInputStream.java | 61 +- .../java/org/apache/hadoop/fs/GetSpaceUsed.java | 147 +++ .../org/apache/hadoop/fs/HarFileSystem.java | 12 +- .../apache/hadoop/fs/PositionedReadable.java | 47 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 8 + .../apache/hadoop/fs/WindowsGetSpaceUsed.java | 46 + .../org/apache/hadoop/http/HttpServer2.java | 29 +- .../apache/hadoop/io/AbstractMapWritable.java | 16 +- .../org/apache/hadoop/metrics2/lib/Interns.java | 16 +- .../apache/hadoop/util/ShutdownHookManager.java | 2 +- .../src/site/markdown/DeprecatedProperties.md | 1 + .../markdown/filesystem/fsdatainputstream.md | 74 +- .../hadoop/crypto/CryptoStreamsTestBase.java | 10 +- .../fs/FileContextMainOperationsBaseTest.java | 3 +- .../test/java/org/apache/hadoop/fs/TestDU.java | 46 +- .../org/apache/hadoop/fs/TestGetSpaceUsed.java | 133 +++ .../fs/contract/AbstractContractAppendTest.java | 5 +- .../fs/contract/AbstractContractConcatTest.java | 2 +- .../fs/contract/AbstractContractCreateTest.java | 10 +- .../fs/contract/AbstractContractDeleteTest.java | 18 +- .../fs/contract/AbstractContractMkdirTest.java | 9 +- .../fs/contract/AbstractContractOpenTest.java | 4 +- .../fs/contract/AbstractContractRenameTest.java | 35 +- .../AbstractContractRootDirectoryTest.java | 14 +- .../fs/contract/AbstractContractSeekTest.java | 259 ++++- .../fs/contract/AbstractFSContractTestBase.java | 2 +- .../hadoop/fs/contract/ContractOptions.java | 49 +- .../hadoop/fs/contract/ContractTestUtils.java | 25 +- .../org/apache/hadoop/http/TestHttpServer.java | 10 + .../src/test/resources/contract/localfs.xml | 4 +- .../hadoop-kms/src/site/markdown/index.md.vm | 39 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 8 + .../apache/hadoop/hdfs/HdfsConfiguration.java | 3 - .../hdfs/client/HdfsClientConfigKeys.java | 6 - .../hadoop/hdfs/web/ByteRangeInputStream.java | 22 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 18 - .../server/blockmanagement/DatanodeManager.java | 115 +- .../hadoop/hdfs/server/datanode/DNConf.java | 18 + .../hadoop/hdfs/server/datanode/DataNode.java | 9 + .../datanode/fsdataset/impl/BlockPoolSlice.java | 93 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 20 +- .../hdfs/server/namenode/FSNamesystem.java | 78 +- .../hadoop/hdfs/server/namenode/NameNode.java | 24 +- .../src/main/resources/hdfs-default.xml | 1033 +++++++++++++++++- .../hdfs/TestDecommissionWithStriped.java | 76 +- .../TestSortLocatedStripedBlock.java | 557 ++++++++++ .../server/datanode/TestBlockReplacement.java | 3 +- .../TestDataNodeVolumeFailureToleration.java | 19 +- .../fsdataset/impl/TestSpaceReservation.java | 60 + .../namenode/TestNameNodeReconfigure.java | 51 +- .../hadoop/tools/TestHdfsConfigFields.java | 87 +- .../org/apache/hadoop/tools/TestJMXGet.java | 3 - .../hadoop/mapreduce/task/reduce/Fetcher.java | 2 +- .../mapreduce/task/reduce/TestFetcher.java | 37 + .../apache/hadoop/fs/s3a/S3AInputStream.java | 271 ++++- .../hadoop/fs/s3a/TestS3AConfiguration.java | 2 + .../fs/s3a/scale/TestS3ADeleteManyFiles.java | 9 + .../hadoop/fs/azure/NativeAzureFileSystem.java | 4 +- .../src/main/data/2jobs2min-rumen-jh.json | 6 - .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 - .../yarn/sls/scheduler/RMNodeWrapper.java | 9 - .../hadoop/yarn/api/records/ContainerId.java | 11 +- .../hadoop/yarn/conf/YarnConfiguration.java | 9 - .../yarn/client/api/impl/NMClientImpl.java | 36 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 10 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 2 +- .../webapps/static/dt-1.9.4/css/jui-dt.css | 4 +- .../src/main/resources/yarn-default.xml | 13 - .../hadoop/yarn/conf/TestYarnConfiguration.java | 8 +- .../yarn/nodelabels/NodeLabelTestBase.java | 2 +- .../apache/hadoop/yarn/util/TestFSDownload.java | 2 +- .../client/impl/zk/RegistrySecurity.java | 4 +- .../nodemanager/NodeStatusUpdaterImpl.java | 1 + .../launcher/ContainerLaunch.java | 2 +- .../localizer/LocalizedResource.java | 8 +- .../nodemanager/util/ProcessIdFileReader.java | 2 +- .../nodemanager/webapp/ApplicationPage.java | 18 +- .../nodemanager/TestNodeManagerResync.java | 33 +- .../TestLogAggregationService.java | 19 +- .../nodemanager/webapp/TestNMAppsPage.java | 86 ++ .../server/resourcemanager/AdminService.java | 6 +- .../resourcemanager/NodesListManager.java | 104 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../resourcemanager/ResourceTrackerService.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 24 +- .../server/resourcemanager/rmnode/RMNode.java | 4 - .../resourcemanager/rmnode/RMNodeImpl.java | 22 +- .../scheduler/AppSchedulingInfo.java | 12 +- .../scheduler/capacity/AbstractCSQueue.java | 14 +- .../capacity/CapacityHeadroomProvider.java | 28 +- .../scheduler/capacity/CapacityScheduler.java | 6 + .../capacity/CapacitySchedulerContext.java | 12 +- .../scheduler/capacity/LeafQueue.java | 51 +- .../scheduler/policy/FairOrderingPolicy.java | 3 +- .../yarn/server/resourcemanager/MockNodes.java | 9 - .../yarn/server/resourcemanager/TestRMHA.java | 42 +- .../TestResourceTrackerService.java | 216 +--- .../applicationsmanager/TestAMRestart.java | 107 ++ ...estProportionalCapacityPreemptionPolicy.java | 6 +- ...pacityPreemptionPolicyForNodePartitions.java | 14 +- .../capacity/TestApplicationLimits.java | 22 +- .../TestApplicationLimitsByPartition.java | 202 +++- .../scheduler/capacity/TestLeafQueue.java | 10 +- .../scheduler/capacity/TestReservations.java | 5 + .../scheduler/capacity/TestUtils.java | 12 +- .../webapp/TestRMWebServicesNodes.java | 12 +- .../hadoop/yarn/server/MiniYARNCluster.java | 2 +- .../timeline/EntityGroupFSTimelineStore.java | 6 + .../TestEntityGroupFSTimelineStore.java | 55 +- .../yarn/server/timeline/TestLogInfo.java | 28 +- .../server/webproxy/WebAppProxyServlet.java | 4 +- .../src/site/markdown/CapacityScheduler.md | 28 + 122 files changed, 4422 insertions(+), 1328 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae6c0e83/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ----------------------------------------------------------------------
