Merge remote-tracking branch 'origin/trunk' into HDFS-8966
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27009cf3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27009cf3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27009cf3 Branch: refs/heads/HDFS-8966 Commit: 27009cf32aa8044e38f8d0761d8193c2010c87ca Parents: 08aafac 7e28296 Author: Jing Zhao <[email protected]> Authored: Tue Nov 3 13:07:04 2015 -0800 Committer: Jing Zhao <[email protected]> Committed: Tue Nov 3 13:07:04 2015 -0800 ---------------------------------------------------------------------- .../main/resources/assemblies/hadoop-dist.xml | 4 +- .../assemblies/hadoop-hdfs-nfs-dist.xml | 4 +- .../resources/assemblies/hadoop-httpfs-dist.xml | 4 +- .../resources/assemblies/hadoop-kms-dist.xml | 4 +- .../assemblies/hadoop-mapreduce-dist.xml | 4 +- .../resources/assemblies/hadoop-nfs-dist.xml | 4 +- .../main/resources/assemblies/hadoop-sls.xml | 4 +- .../main/resources/assemblies/hadoop-src.xml | 4 +- .../main/resources/assemblies/hadoop-tools.xml | 4 +- .../resources/assemblies/hadoop-yarn-dist.xml | 4 +- hadoop-common-project/hadoop-common/CHANGES.txt | 32 +- hadoop-common-project/hadoop-common/pom.xml | 1 - .../java/org/apache/hadoop/fs/FileSystem.java | 19 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 8 +- .../org/apache/hadoop/fs/HarFileSystem.java | 6 + .../rawcoder/AbstractRawErasureCoder.java | 107 +- .../rawcoder/AbstractRawErasureDecoder.java | 8 +- .../rawcoder/AbstractRawErasureEncoder.java | 8 +- .../io/erasurecode/rawcoder/CoderOption.java | 43 + .../io/erasurecode/rawcoder/RSRawDecoder.java | 4 +- .../io/erasurecode/rawcoder/RSRawEncoder.java | 48 +- .../erasurecode/rawcoder/RawErasureCoder.java | 23 +- .../erasurecode/rawcoder/RawErasureDecoder.java | 32 +- .../erasurecode/rawcoder/RawErasureEncoder.java | 35 +- .../io/erasurecode/rawcoder/XORRawDecoder.java | 1 - .../io/erasurecode/rawcoder/XORRawEncoder.java | 1 - .../apache/hadoop/io/retry/RetryPolicies.java | 44 +- .../org/apache/hadoop/ipc/CallerContext.java | 13 +- .../java/org/apache/hadoop/util/ProtoUtil.java | 2 +- .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../hadoop/io/erasurecode/TestCoderBase.java | 34 +- .../erasurecode/rawcoder/TestRawCoderBase.java | 37 +- .../erasurecode/rawcoder/TestXORRawCoder.java | 1 + .../apache/hadoop/io/retry/TestRetryProxy.java | 27 +- .../io/retry/UnreliableImplementation.java | 17 + .../hadoop/io/retry/UnreliableInterface.java | 3 + .../java/org/apache/hadoop/net/TestDNS.java | 12 +- hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 21 + .../hadoop/hdfs/DFSStripedInputStream.java | 1 - .../hadoop/hdfs/DFSStripedOutputStream.java | 21 +- .../datatransfer/ReplaceDatanodeOnFailure.java | 78 +- .../hadoop/fs/TestUrlStreamHandlerFactory.java | 80 + .../java/org/apache/hadoop/fs/TestXAttr.java | 94 + .../org/apache/hadoop/hdfs/TestDFSPacket.java | 94 + .../hadoop/hdfs/TestDefaultNameNodePort.java | 66 + .../org/apache/hadoop/hdfs/TestPeerCache.java | 292 +++ .../hdfs/client/impl/TestLeaseRenewer.java | 207 +++ .../hadoop/hdfs/protocol/TestExtendedBlock.java | 77 + .../hdfs/shortcircuit/TestShortCircuitShm.java | 109 ++ .../hadoop/hdfs/util/TestByteArrayManager.java | 644 +++++++ .../hdfs/util/TestExactSizeInputStream.java | 129 ++ .../hdfs/web/TestByteRangeInputStream.java | 292 +++ .../hdfs/web/TestOffsetUrlInputStream.java | 64 + .../apache/hadoop/hdfs/web/TestTokenAspect.java | 316 ++++ .../hdfs/web/TestURLConnectionFactory.java | 50 + .../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 217 +++ .../hdfs/web/TestWebHdfsContentLength.java | 197 +++ .../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 + ...ClientCredentialTimeBasedTokenRefresher.java | 139 ++ ...TestRefreshTokenTimeBasedTokenRefresher.java | 138 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 69 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 - .../server/blockmanagement/BlockManager.java | 5 + .../blockmanagement/BlockPlacementPolicy.java | 8 +- .../BlockPlacementPolicyDefault.java | 46 +- .../BlockPlacementPolicyWithNodeGroup.java | 78 +- .../hadoop/hdfs/server/common/Storage.java | 15 + .../erasurecode/ErasureCodingWorker.java | 7 +- .../hadoop/hdfs/server/namenode/FSImage.java | 4 + .../hdfs/server/namenode/FSNamesystem.java | 17 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 25 + .../hdfs/server/namenode/NameNodeMXBean.java | 5 + .../hdfs/server/namenode/ha/EditLogTailer.java | 4 +- .../hadoop/fs/TestUrlStreamHandlerFactory.java | 80 - .../java/org/apache/hadoop/fs/TestXAttr.java | 94 - .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 22 + .../apache/hadoop/hdfs/StripedFileTestUtil.java | 8 +- .../apache/hadoop/hdfs/TestAclsEndToEnd.java | 1657 ++++++++++++++++++ .../apache/hadoop/hdfs/TestCrcCorruption.java | 120 +- .../org/apache/hadoop/hdfs/TestDFSPacket.java | 94 - .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 + .../TestDFSStripedOutputStreamWithFailure.java | 33 +- ...estDFSStripedOutputStreamWithFailure020.java | 22 + ...estDFSStripedOutputStreamWithFailure030.java | 22 + ...estDFSStripedOutputStreamWithFailure040.java | 22 + ...estDFSStripedOutputStreamWithFailure050.java | 22 + ...estDFSStripedOutputStreamWithFailure060.java | 22 + ...estDFSStripedOutputStreamWithFailure070.java | 22 + ...estDFSStripedOutputStreamWithFailure080.java | 22 + ...estDFSStripedOutputStreamWithFailure090.java | 22 + ...estDFSStripedOutputStreamWithFailure100.java | 22 + ...estDFSStripedOutputStreamWithFailure110.java | 22 + ...estDFSStripedOutputStreamWithFailure120.java | 22 + ...estDFSStripedOutputStreamWithFailure130.java | 22 + ...estDFSStripedOutputStreamWithFailure140.java | 22 + ...estDFSStripedOutputStreamWithFailure150.java | 22 + ...estDFSStripedOutputStreamWithFailure160.java | 22 + ...estDFSStripedOutputStreamWithFailure170.java | 22 + ...estDFSStripedOutputStreamWithFailure180.java | 22 + ...estDFSStripedOutputStreamWithFailure190.java | 22 + ...estDFSStripedOutputStreamWithFailure200.java | 22 + ...estDFSStripedOutputStreamWithFailure210.java | 23 + .../hadoop/hdfs/TestDefaultNameNodePort.java | 68 - .../hadoop/hdfs/TestDistributedFileSystem.java | 28 + .../TestErasureCodingPolicyWithSnapshot.java | 199 +++ .../apache/hadoop/hdfs/TestLeaseRecovery.java | 35 +- .../org/apache/hadoop/hdfs/TestPeerCache.java | 293 ---- .../TestReadStripedFileWithMissingBlocks.java | 6 +- .../hadoop/hdfs/TestRecoverStripedFile.java | 143 +- .../org/apache/hadoop/hdfs/TestReplication.java | 115 +- .../hdfs/TestSafeModeWithStripedFile.java | 5 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 8 +- .../hdfs/TestWriteStripedFileWithFailure.java | 6 +- .../hdfs/client/impl/TestLeaseRenewer.java | 209 --- .../hadoop/hdfs/protocol/TestExtendedBlock.java | 77 - .../hdfs/server/balancer/TestBalancer.java | 572 +++--- .../TestBlockTokenWithDFSStriped.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 31 + .../TestReplicationPolicyWithNodeGroup.java | 58 + .../server/datanode/FsDatasetTestUtils.java | 7 + .../fsdataset/impl/FsDatasetImplTestUtils.java | 25 + .../hdfs/server/namenode/TestAuditLogger.java | 41 +- .../server/namenode/TestNameNodeMXBean.java | 62 +- .../namenode/TestRecoverStripedBlocks.java | 70 + .../namenode/ha/TestBootstrapStandby.java | 79 +- .../hdfs/shortcircuit/TestShortCircuitShm.java | 109 -- .../hadoop/hdfs/util/TestByteArrayManager.java | 645 ------- .../hdfs/util/TestExactSizeInputStream.java | 129 -- .../hdfs/web/TestByteRangeInputStream.java | 292 --- .../hdfs/web/TestOffsetUrlInputStream.java | 64 - .../apache/hadoop/hdfs/web/TestTokenAspect.java | 316 ---- .../hdfs/web/TestURLConnectionFactory.java | 50 - .../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 216 --- .../hdfs/web/TestWebHdfsContentLength.java | 197 --- .../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 - ...ClientCredentialTimeBasedTokenRefresher.java | 139 -- ...TestRefreshTokenTimeBasedTokenRefresher.java | 138 -- hadoop-mapreduce-project/CHANGES.txt | 48 +- .../apache/hadoop/mapreduce/v2/app/job/Job.java | 2 + .../mapreduce/v2/app/job/impl/JobImpl.java | 13 +- .../v2/app/rm/RMContainerAllocator.java | 9 + .../jobhistory/TestJobHistoryEventHandler.java | 10 +- .../hadoop/mapreduce/v2/app/MockJobs.java | 6 + .../mapreduce/v2/app/TestRuntimeEstimators.java | 6 + .../mapreduce/v2/app/job/impl/TestJobImpl.java | 35 + .../v2/app/rm/TestRMContainerAllocator.java | 2 + .../apache/hadoop/mapreduce/TypeConverter.java | 34 +- .../mapreduce/v2/api/records/JobReport.java | 4 + .../v2/api/records/impl/pb/JobReportPBImpl.java | 44 +- .../mapreduce/v2/util/MRBuilderUtils.java | 15 +- .../src/main/proto/mr_protos.proto | 1 + .../hadoop/mapreduce/TestTypeConverter.java | 8 +- .../org/apache/hadoop/mapred/JobPriority.java | 5 +- .../apache/hadoop/mapreduce/JobPriority.java | 4 +- .../hadoop/mapreduce/v2/hs/CompletedJob.java | 7 + .../mapreduce/v2/hs/HistoryFileManager.java | 13 +- .../hadoop/mapreduce/v2/hs/PartialJob.java | 7 + .../v2/hs/webapp/TestHsWebServicesAcls.java | 5 + .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 41 +- hadoop-project/pom.xml | 10 + .../fs/azure/AzureNativeFileSystemStore.java | 32 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 125 +- .../hadoop/fs/azure/SelfRenewingLease.java | 5 +- ...estFileSystemOperationExceptionHandling.java | 131 ++ ...perationsExceptionHandlingMultiThreaded.java | 185 ++ .../fs/azure/TestNativeAzureFileSystemLive.java | 86 + .../tools/mapred/lib/DynamicInputChunk.java | 137 +- .../mapred/lib/DynamicInputChunkContext.java | 113 ++ .../tools/mapred/lib/DynamicInputFormat.java | 31 +- .../tools/mapred/lib/DynamicRecordReader.java | 13 +- .../org/apache/hadoop/tools/StubContext.java | 4 + .../mapred/lib/TestDynamicInputFormat.java | 33 +- .../hadoop-sls/src/main/assemblies/sls.xml | 4 +- hadoop-yarn-project/CHANGES.txt | 62 +- .../distributedshell/TestDistributedShell.java | 9 +- .../org/apache/hadoop/yarn/client/RMProxy.java | 6 +- .../ApplicationHistoryServer.java | 2 +- .../nodemanager/DefaultContainerExecutor.java | 9 +- .../nodemanager/DockerContainerExecutor.java | 9 +- .../localizer/LocalResourcesTrackerImpl.java | 10 + .../localizer/ResourceLocalizationService.java | 13 + .../impl/container-executor.c | 9 +- .../test/test-container-executor.c | 11 + .../TestLocalResourcesTrackerImpl.java | 6 +- .../TestResourceLocalizationService.java | 293 +++- .../server/resourcemanager/RMAppManager.java | 8 +- .../metrics/SystemMetricsPublisher.java | 4 +- .../recovery/ZKRMStateStore.java | 25 +- .../scheduler/fair/FSAppAttempt.java | 16 +- .../metrics/TestSystemMetricsPublisher.java | 2 +- .../recovery/TestZKRMStateStore.java | 68 + 191 files changed, 8716 insertions(+), 4409 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27009cf3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27009cf3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27009cf3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java ----------------------------------------------------------------------
