Merge branch 'trunk' into HDFS-8707
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbba8701 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbba8701 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbba8701 Branch: refs/heads/HDFS-8707 Commit: fbba87017d3337c3e8a3450b333a6e2e3db69f32 Parents: 457fe08 0ca8df7 Author: Haohui Mai <whe...@apache.org> Authored: Thu Nov 12 10:21:16 2015 -0800 Committer: Haohui Mai <whe...@apache.org> Committed: Thu Nov 12 10:21:16 2015 -0800 ---------------------------------------------------------------------- BUILDING.txt | 25 + dev-support/docker/Dockerfile | 89 +- .../main/resources/assemblies/hadoop-dist.xml | 4 +- .../assemblies/hadoop-hdfs-nfs-dist.xml | 4 +- .../resources/assemblies/hadoop-httpfs-dist.xml | 4 +- .../resources/assemblies/hadoop-kms-dist.xml | 4 +- .../assemblies/hadoop-mapreduce-dist.xml | 4 +- .../resources/assemblies/hadoop-nfs-dist.xml | 4 +- .../main/resources/assemblies/hadoop-sls.xml | 4 +- .../main/resources/assemblies/hadoop-src.xml | 4 +- .../main/resources/assemblies/hadoop-tools.xml | 4 +- .../resources/assemblies/hadoop-yarn-dist.xml | 4 +- hadoop-common-project/hadoop-common/CHANGES.txt | 84 +- hadoop-common-project/hadoop-common/pom.xml | 22 +- .../hadoop-common/src/CMakeLists.txt | 25 + .../hadoop-common/src/config.h.cmake | 1 + .../hadoop-common/src/main/bin/hadoop | 26 +- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 6 +- .../src/main/bin/hadoop-daemons.sh | 6 +- .../src/main/bin/hadoop-functions.sh | 66 +- .../src/main/bin/hadoop-layout.sh.example | 16 +- .../hadoop-common/src/main/bin/rcc | 4 +- .../hadoop-common/src/main/bin/slaves.sh | 6 +- .../hadoop-common/src/main/bin/start-all.sh | 6 +- .../hadoop-common/src/main/bin/stop-all.sh | 6 +- .../main/conf/hadoop-user-functions.sh.example | 10 +- .../java/org/apache/hadoop/fs/FileSystem.java | 19 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 8 +- .../org/apache/hadoop/fs/HarFileSystem.java | 6 + .../io/erasurecode/ErasureCodeNative.java | 86 ++ .../rawcoder/AbstractRawErasureCoder.java | 107 +- .../rawcoder/AbstractRawErasureDecoder.java | 8 +- .../rawcoder/AbstractRawErasureEncoder.java | 8 +- .../io/erasurecode/rawcoder/CoderOption.java | 43 + .../erasurecode/rawcoder/DummyRawDecoder.java | 47 + .../erasurecode/rawcoder/DummyRawEncoder.java | 46 + .../rawcoder/DummyRawErasureCoderFactory.java | 36 + .../io/erasurecode/rawcoder/RSRawDecoder.java | 4 +- .../io/erasurecode/rawcoder/RSRawEncoder.java | 48 +- .../erasurecode/rawcoder/RawErasureCoder.java | 23 +- .../erasurecode/rawcoder/RawErasureDecoder.java | 32 +- .../erasurecode/rawcoder/RawErasureEncoder.java | 35 +- .../io/erasurecode/rawcoder/XORRawDecoder.java | 1 - .../io/erasurecode/rawcoder/XORRawEncoder.java | 1 - .../apache/hadoop/io/retry/RetryPolicies.java | 44 +- .../org/apache/hadoop/ipc/CallerContext.java | 13 +- .../metrics2/impl/MetricsSourceAdapter.java | 11 +- .../org/apache/hadoop/security/token/Token.java | 11 +- .../apache/hadoop/util/NativeCodeLoader.java | 11 +- .../hadoop/util/NativeLibraryChecker.java | 20 +- .../java/org/apache/hadoop/util/ProtoUtil.java | 2 +- .../src/main/native/native.vcxproj | 22 +- .../io/erasurecode/coder/erasure_code_native.c | 49 + ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 + .../apache/hadoop/io/erasurecode/erasure_code.c | 271 +++++ .../io/erasurecode/include/erasure_code.h | 125 +++ .../hadoop/io/erasurecode/include/gf_util.h | 111 ++ .../org/apache/hadoop/net/unix/DomainSocket.c | 18 +- .../JniBasedUnixGroupsNetgroupMapping.c | 18 +- .../org/apache/hadoop/util/NativeCodeLoader.c | 10 + .../hadoop/io/erasurecode/erasure_code_test.c | 310 ++++++ .../src/main/resources/core-default.xml | 10 +- .../hadoop-common/src/site/markdown/Metrics.md | 1 + .../test/aop/org/apache/hadoop/fi/FiConfig.java | 50 - .../org/apache/hadoop/fi/ProbabilityModel.java | 108 -- .../hadoop/io/erasurecode/TestCoderBase.java | 38 +- .../erasurecode/rawcoder/TestDummyRawCoder.java | 83 ++ .../erasurecode/rawcoder/TestRawCoderBase.java | 45 +- .../erasurecode/rawcoder/TestXORRawCoder.java | 1 + .../apache/hadoop/io/retry/TestRetryProxy.java | 27 +- .../io/retry/UnreliableImplementation.java | 17 + .../hadoop/io/retry/UnreliableInterface.java | 3 + .../metrics2/impl/TestMetricsSourceAdapter.java | 195 +++- .../java/org/apache/hadoop/net/TestDNS.java | 12 +- .../scripts/hadoop_add_common_to_classpath.bats | 4 +- .../hadoop_add_to_classpath_toolspath.bats | 74 ++ .../src/test/scripts/hadoop_basic_init.bats | 2 +- .../hadoop-kms/src/main/sbin/kms.sh | 6 +- .../hadoop/hdfs/DFSStripedInputStream.java | 1 - .../hadoop/hdfs/DFSStripedOutputStream.java | 21 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 22 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 2 +- .../hadoop/hdfs/NameNodeProxiesClient.java | 2 +- .../hadoop/hdfs/util/ByteArrayManager.java | 63 +- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 26 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 12 + .../hadoop/hdfs/web/resources/PutOpParam.java | 2 + .../hadoop/hdfs/util/TestByteArrayManager.java | 2 +- .../hdfs/web/TestWebHdfsContentLength.java | 23 +- .../src/test/resources/log4j.properties | 49 + .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 6 +- .../hadoop-hdfs-native-client/pom.xml | 46 +- .../src/CMakeLists.txt | 8 +- .../main/native/libhdfs-tests/CMakeLists.txt | 1 + .../src/main/native/libhdfs/CMakeLists.txt | 17 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 22 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 165 ++- .../dev-support/findbugsExcludeFile.xml | 7 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - .../src/main/bin/distribute-exclude.sh | 4 +- .../hadoop-hdfs/src/main/bin/hdfs | 19 +- .../src/main/bin/refresh-namenodes.sh | 6 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 6 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 6 +- .../src/main/bin/start-secure-dns.sh | 6 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 6 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 6 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 6 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 65 +- .../hdfs/server/blockmanagement/BlockInfo.java | 2 +- .../server/blockmanagement/BlockManager.java | 9 + .../blockmanagement/BlockPlacementPolicy.java | 61 +- .../BlockPlacementPolicyDefault.java | 119 +- .../BlockPlacementPolicyWithNodeGroup.java | 113 +- .../BlockPlacementPolicyWithUpgradeDomain.java | 84 +- .../server/blockmanagement/DatanodeManager.java | 14 +- .../hadoop/hdfs/server/common/Storage.java | 15 + .../server/datanode/BlockRecoveryWorker.java | 69 +- .../hdfs/server/datanode/DataStorage.java | 12 - .../erasurecode/ErasureCodingWorker.java | 7 +- .../web/webhdfs/DataNodeUGIProvider.java | 106 +- .../datanode/web/webhdfs/WebHdfsHandler.java | 2 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 4 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 3 +- .../hdfs/server/namenode/FSDirectory.java | 8 +- .../hadoop/hdfs/server/namenode/FSImage.java | 4 + .../hdfs/server/namenode/FSNamesystem.java | 73 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 4 +- .../hadoop/hdfs/server/namenode/NNStorage.java | 25 + .../hadoop/hdfs/server/namenode/NameNode.java | 13 +- .../hdfs/server/namenode/NameNodeMXBean.java | 5 + .../hdfs/server/namenode/NamenodeFsck.java | 9 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 4 +- .../web/resources/NamenodeWebHdfsMethods.java | 8 + .../server/protocol/ReplicaRecoveryInfo.java | 6 + .../org/apache/hadoop/hdfs/web/JsonUtil.java | 15 + .../src/main/resources/hdfs-default.xml | 17 +- .../src/main/webapps/hdfs/dfshealth.html | 4 +- .../src/main/webapps/static/hadoop.css | 4 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 84 +- .../apache/hadoop/fi/DataTransferTestUtil.java | 485 -------- .../test/aop/org/apache/hadoop/fi/FiConfig.java | 55 - .../org/apache/hadoop/fi/FiHFlushTestUtil.java | 66 -- .../aop/org/apache/hadoop/fi/FiTestUtil.java | 209 ---- .../test/aop/org/apache/hadoop/fi/Pipeline.java | 50 - .../aop/org/apache/hadoop/fi/PipelineTest.java | 27 - .../org/apache/hadoop/fi/ProbabilityModel.java | 107 -- .../org/apache/hadoop/fs/TestFiListPath.java | 105 -- .../aop/org/apache/hadoop/fs/TestFiRename.java | 272 ----- .../org/apache/hadoop/hdfs/DFSClientAspects.aj | 103 -- .../aop/org/apache/hadoop/hdfs/HFlushAspects.aj | 64 -- .../apache/hadoop/hdfs/PipelinesTestUtil.java | 153 --- .../org/apache/hadoop/hdfs/TestFiHFlush.java | 180 --- .../org/apache/hadoop/hdfs/TestFiPipelines.java | 247 ----- .../hdfs/protocol/ClientProtocolAspects.aj | 38 - .../server/datanode/BlockReceiverAspects.aj | 232 ---- .../datanode/DataTransferProtocolAspects.aj | 80 -- .../hdfs/server/datanode/FSDatasetAspects.aj | 63 -- .../datanode/TestFiDataTransferProtocol.java | 312 ------ .../datanode/TestFiDataTransferProtocol2.java | 289 ----- .../server/datanode/TestFiPipelineClose.java | 250 ----- .../server/namenode/FileDataServletAspects.aj | 46 - .../hdfs/server/namenode/ListPathAspects.aj | 48 - .../hdfs/server/namenode/RenameAspects.aj | 66 -- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 36 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 24 + .../apache/hadoop/hdfs/StripedFileTestUtil.java | 8 +- .../apache/hadoop/hdfs/TestAclsEndToEnd.java | 1042 +++++++++++++++++- .../apache/hadoop/hdfs/TestCrcCorruption.java | 120 +- .../hadoop/hdfs/TestDFSClientFailover.java | 35 + .../hadoop/hdfs/TestDFSStripedOutputStream.java | 2 + .../TestDFSStripedOutputStreamWithFailure.java | 33 +- ...estDFSStripedOutputStreamWithFailure020.java | 22 + ...estDFSStripedOutputStreamWithFailure030.java | 22 + ...estDFSStripedOutputStreamWithFailure040.java | 22 + ...estDFSStripedOutputStreamWithFailure050.java | 22 + ...estDFSStripedOutputStreamWithFailure060.java | 22 + ...estDFSStripedOutputStreamWithFailure070.java | 22 + ...estDFSStripedOutputStreamWithFailure080.java | 22 + ...estDFSStripedOutputStreamWithFailure090.java | 22 + ...estDFSStripedOutputStreamWithFailure100.java | 22 + ...estDFSStripedOutputStreamWithFailure110.java | 22 + ...estDFSStripedOutputStreamWithFailure120.java | 22 + ...estDFSStripedOutputStreamWithFailure130.java | 22 + ...estDFSStripedOutputStreamWithFailure140.java | 22 + ...estDFSStripedOutputStreamWithFailure150.java | 22 + ...estDFSStripedOutputStreamWithFailure160.java | 22 + ...estDFSStripedOutputStreamWithFailure170.java | 22 + ...estDFSStripedOutputStreamWithFailure180.java | 22 + ...estDFSStripedOutputStreamWithFailure190.java | 22 + ...estDFSStripedOutputStreamWithFailure200.java | 22 + ...estDFSStripedOutputStreamWithFailure210.java | 23 + .../hadoop/hdfs/TestDistributedFileSystem.java | 28 + .../TestErasureCodingPolicyWithSnapshot.java | 199 ++++ .../apache/hadoop/hdfs/TestLeaseRecovery.java | 35 +- .../org/apache/hadoop/hdfs/TestPipelines.java | 6 +- .../java/org/apache/hadoop/hdfs/TestQuota.java | 55 +- .../TestReadStripedFileWithMissingBlocks.java | 6 +- .../hadoop/hdfs/TestRecoverStripedFile.java | 143 +-- .../org/apache/hadoop/hdfs/TestReplication.java | 115 +- .../hdfs/TestSafeModeWithStripedFile.java | 5 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 8 +- .../hdfs/TestWriteStripedFileWithFailure.java | 6 +- .../hdfs/server/balancer/TestBalancer.java | 645 ++++++----- .../blockmanagement/TestBlockManager.java | 13 +- .../TestBlockTokenWithDFSStriped.java | 4 +- .../blockmanagement/TestReplicationPolicy.java | 122 +- .../TestReplicationPolicyConsiderLoad.java | 56 + .../TestReplicationPolicyWithNodeGroup.java | 64 +- .../TestReplicationPolicyWithUpgradeDomain.java | 171 ++- .../datanode/FsDatasetImplTestUtilsFactory.java | 5 + .../server/datanode/FsDatasetTestUtils.java | 33 + .../hdfs/server/datanode/TestBlockRecovery.java | 37 + .../TestDataNodeMultipleRegistrations.java | 8 +- .../fsdataset/impl/FsDatasetImplTestUtils.java | 51 + .../impl/TestInterDatanodeProtocol.java | 5 +- .../web/webhdfs/TestDataNodeUGIProvider.java | 231 ---- .../server/namenode/NNThroughputBenchmark.java | 6 +- .../hdfs/server/namenode/TestAuditLogger.java | 37 +- .../hdfs/server/namenode/TestBackupNode.java | 71 ++ .../TestCommitBlockSynchronization.java | 4 +- .../TestCommitBlockWithInvalidGenStamp.java | 100 ++ .../server/namenode/TestNameNodeMXBean.java | 64 +- .../namenode/TestNamenodeCapacityReport.java | 14 +- .../namenode/TestQuotaWithStripedBlocks.java | 4 +- .../namenode/TestRecoverStripedBlocks.java | 70 ++ .../namenode/ha/TestBootstrapStandby.java | 79 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 10 +- .../hdfs/server/namenode/ha/TestHAMetrics.java | 56 + .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 80 ++ hadoop-mapreduce-project/CHANGES.txt | 74 +- hadoop-mapreduce-project/bin/mapred | 25 +- .../bin/mr-jobhistory-daemon.sh | 6 +- .../apache/hadoop/mapreduce/v2/app/job/Job.java | 2 + .../mapreduce/v2/app/job/impl/JobImpl.java | 13 +- .../v2/app/rm/RMContainerAllocator.java | 9 + .../mapreduce/v2/app/webapp/TaskPage.java | 2 +- .../mapreduce/v2/app/webapp/TasksPage.java | 2 +- .../jobhistory/TestJobHistoryEventHandler.java | 10 +- .../hadoop/mapreduce/v2/app/MockJobs.java | 6 + .../mapreduce/v2/app/TestRuntimeEstimators.java | 6 + .../mapreduce/v2/app/job/impl/TestJobImpl.java | 35 + .../v2/app/rm/TestRMContainerAllocator.java | 2 + .../apache/hadoop/mapreduce/TypeConverter.java | 34 +- .../mapreduce/v2/api/records/JobReport.java | 4 + .../v2/api/records/impl/pb/JobReportPBImpl.java | 44 +- .../mapreduce/v2/util/MRBuilderUtils.java | 15 +- .../src/main/proto/mr_protos.proto | 1 + .../hadoop/mapreduce/TestTypeConverter.java | 8 +- .../org/apache/hadoop/mapred/JobPriority.java | 5 +- .../apache/hadoop/mapreduce/JobPriority.java | 4 +- .../TestClientDistributedCacheManager.java | 40 +- .../hadoop/mapreduce/v2/hs/CompletedJob.java | 7 + .../mapreduce/v2/hs/HistoryFileManager.java | 13 +- .../hadoop/mapreduce/v2/hs/PartialJob.java | 7 + .../v2/hs/webapp/TestHsWebServicesAcls.java | 5 + .../mapred/TestMRTimelineEventHandling.java | 10 + .../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 41 +- .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- hadoop-project-dist/pom.xml | 13 +- hadoop-project/pom.xml | 6 + .../s3a/BlockingThreadPoolExecutorService.java | 272 +++++ .../org/apache/hadoop/fs/s3a/Constants.java | 13 +- .../hadoop/fs/s3a/S3AFastOutputStream.java | 4 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 82 +- .../src/site/markdown/tools/hadoop-aws/index.md | 10 +- .../TestBlockingThreadPoolExecutorService.java | 182 +++ .../fs/s3a/TestS3ABlockingThreadPool.java | 80 ++ .../fs/azure/AzureNativeFileSystemStore.java | 32 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 125 ++- .../hadoop/fs/azure/SelfRenewingLease.java | 5 +- ...estFileSystemOperationExceptionHandling.java | 131 +++ ...perationsExceptionHandlingMultiThreaded.java | 185 ++++ .../fs/azure/TestNativeAzureFileSystemLive.java | 86 ++ .../TestAzureFileSystemInstrumentation.java | 25 +- .../tools/mapred/lib/DynamicInputChunk.java | 137 +-- .../mapred/lib/DynamicInputChunkContext.java | 113 ++ .../tools/mapred/lib/DynamicInputFormat.java | 31 +- .../tools/mapred/lib/DynamicRecordReader.java | 13 +- .../org/apache/hadoop/tools/StubContext.java | 4 + .../mapred/lib/TestDynamicInputFormat.java | 33 +- .../src/main/native/pipes/impl/HadoopPipes.cc | 4 +- .../main/native/utils/api/hadoop/SerialUtils.hh | 1 + .../src/main/native/utils/impl/SerialUtils.cc | 7 + .../hadoop-sls/src/main/assemblies/sls.xml | 4 +- .../hadoop-sls/src/main/bin/rumen2sls.sh | 9 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 12 +- hadoop-yarn-project/CHANGES.txt | 83 +- .../hadoop-yarn/bin/start-yarn.sh | 6 +- .../hadoop-yarn/bin/stop-yarn.sh | 6 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 16 +- .../hadoop-yarn/bin/yarn-daemon.sh | 6 +- .../hadoop-yarn/bin/yarn-daemons.sh | 6 +- .../distributedshell/ApplicationMaster.java | 12 +- .../distributedshell/TestDistributedShell.java | 9 +- .../apache/hadoop/yarn/client/api/NMClient.java | 16 + .../yarn/client/api/async/NMClientAsync.java | 154 ++- .../api/async/impl/NMClientAsyncImpl.java | 127 ++- .../yarn/client/api/impl/NMClientImpl.java | 35 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 3 +- .../api/async/impl/TestNMClientAsync.java | 93 +- .../yarn/client/api/impl/TestNMClient.java | 31 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 32 + .../org/apache/hadoop/yarn/client/RMProxy.java | 6 +- .../hadoop/yarn/webapp/view/JQueryUI.java | 21 +- .../dt-plugin-1.10.7/sorting/natural.js.gz | Bin 0 -> 1448 bytes .../src/main/resources/yarn-default.xml | 8 +- .../ApplicationHistoryServer.java | 2 +- .../webapp/AppAttemptPage.java | 2 +- .../webapp/AppPage.java | 2 +- .../webapp/TestAHSWebApp.java | 50 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 6 +- .../nodemanager/DefaultContainerExecutor.java | 9 +- .../nodemanager/DockerContainerExecutor.java | 9 +- .../localizer/LocalResourcesTrackerImpl.java | 10 + .../localizer/ResourceLocalizationService.java | 13 + .../nodemanager/webapp/AllApplicationsPage.java | 9 +- .../nodemanager/webapp/AllContainersPage.java | 8 +- .../impl/container-executor.c | 9 +- .../test/test-container-executor.c | 11 + .../TestLocalResourcesTrackerImpl.java | 6 +- .../TestResourceLocalizationService.java | 293 ++++- .../server/resourcemanager/RMAppManager.java | 8 +- .../recovery/ZKRMStateStore.java | 25 +- .../scheduler/fair/FSAppAttempt.java | 16 +- .../recovery/TestZKRMStateStore.java | 68 ++ .../resourcemanager/webapp/TestRMWebApp.java | 5 + 328 files changed, 9618 insertions(+), 5884 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbba8701/BUILDING.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbba8701/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt ----------------------------------------------------------------------
