Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7a7fd5e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7a7fd5e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7a7fd5e Branch: refs/heads/HDFS-7240 Commit: e7a7fd5ee1b363945673987783239be9f918c974 Parents: 3100002 a30eccb Author: Anu Engineer <[email protected]> Authored: Fri Nov 20 23:53:35 2015 -0800 Committer: Anu Engineer <[email protected]> Committed: Fri Nov 20 23:53:35 2015 -0800 ---------------------------------------------------------------------- BUILDING.txt | 35 +- dev-support/docker/Dockerfile | 89 ++-- hadoop-common-project/hadoop-common/CHANGES.txt | 71 ++- hadoop-common-project/hadoop-common/pom.xml | 21 +- .../hadoop-common/src/CMakeLists.txt | 25 + .../hadoop-common/src/config.h.cmake | 1 + .../hadoop-common/src/main/bin/hadoop | 11 + .../org/apache/hadoop/http/HttpServer2.java | 3 + .../org/apache/hadoop/io/BytesWritable.java | 4 +- .../io/erasurecode/ErasureCodeNative.java | 86 ++++ .../metrics2/impl/MetricsSourceAdapter.java | 11 +- .../apache/hadoop/security/SaslRpcServer.java | 5 +- .../hadoop/security/authorize/package-info.java | 2 +- .../org/apache/hadoop/tools/package-info.java | 3 - .../apache/hadoop/util/NativeCodeLoader.java | 11 +- .../hadoop/util/NativeLibraryChecker.java | 20 +- .../org/apache/hadoop/util/StringUtils.java | 35 ++ .../src/main/native/native.vcxproj | 22 +- .../io/erasurecode/coder/erasure_code_native.c | 49 ++ ...he_hadoop_io_erasurecode_ErasureCodeNative.h | 29 ++ .../apache/hadoop/io/erasurecode/erasure_code.c | 271 +++++++++++ .../io/erasurecode/include/erasure_code.h | 125 +++++ .../hadoop/io/erasurecode/include/gf_util.h | 111 +++++ .../org/apache/hadoop/net/unix/DomainSocket.c | 18 +- .../org/apache/hadoop/util/NativeCodeLoader.c | 10 + .../hadoop/io/erasurecode/erasure_code_test.c | 310 ++++++++++++ .../src/main/resources/core-default.xml | 19 +- .../test/aop/org/apache/hadoop/fi/FiConfig.java | 50 -- .../org/apache/hadoop/fi/ProbabilityModel.java | 108 ----- .../java/org/apache/hadoop/io/AvroTestUtil.java | 2 +- .../org/apache/hadoop/io/TestArrayFile.java | 17 +- .../hadoop/io/TestArrayPrimitiveWritable.java | 10 +- .../org/apache/hadoop/io/TestArrayWritable.java | 20 +- .../org/apache/hadoop/io/TestBloomMapFile.java | 31 +- .../io/TestBoundedByteArrayOutputStream.java | 12 +- .../hadoop/io/TestDefaultStringifier.java | 10 +- .../apache/hadoop/io/TestEnumSetWritable.java | 15 +- .../apache/hadoop/io/TestGenericWritable.java | 20 +- .../java/org/apache/hadoop/io/TestMD5Hash.java | 14 +- .../org/apache/hadoop/io/TestMapWritable.java | 15 +- .../org/apache/hadoop/io/TestSequenceFile.java | 22 +- .../io/TestSequenceFileSerialization.java | 21 +- .../java/org/apache/hadoop/io/TestSetFile.java | 17 +- .../java/org/apache/hadoop/io/TestText.java | 85 ++-- .../org/apache/hadoop/io/TestTextNonUTF8.java | 12 +- .../java/org/apache/hadoop/io/TestUTF8.java | 16 +- .../apache/hadoop/io/TestVersionedWritable.java | 11 +- .../java/org/apache/hadoop/io/TestWritable.java | 20 +- .../org/apache/hadoop/io/TestWritableName.java | 13 +- .../org/apache/hadoop/io/TestWritableUtils.java | 14 +- .../hadoop/io/compress/TestCodecFactory.java | 14 +- .../io/compress/TestCompressionStreamReuse.java | 10 +- .../hadoop/io/file/tfile/KVGenerator.java | 13 +- .../apache/hadoop/io/file/tfile/KeySampler.java | 4 +- .../apache/hadoop/io/file/tfile/TestTFile.java | 16 +- .../io/file/tfile/TestTFileComparators.java | 14 +- .../hadoop/io/file/tfile/TestTFileSeek.java | 29 +- .../file/tfile/TestTFileSeqFileComparison.java | 25 +- .../hadoop/io/file/tfile/TestTFileSplit.java | 19 +- .../hadoop/io/file/tfile/TestTFileStreams.java | 55 ++- .../file/tfile/TestTFileUnsortedByteArrays.java | 14 +- .../apache/hadoop/io/file/tfile/TestVLong.java | 24 +- .../serializer/avro/TestAvroSerialization.java | 11 +- .../metrics2/impl/TestMetricsSourceAdapter.java | 195 +++++++- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 22 +- .../hadoop/hdfs/DistributedFileSystem.java | 2 +- .../hadoop/hdfs/NameNodeProxiesClient.java | 2 +- .../hadoop/hdfs/util/ByteArrayManager.java | 63 ++- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 26 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 12 + .../hadoop/hdfs/web/resources/PutOpParam.java | 2 + .../hadoop/hdfs/util/TestByteArrayManager.java | 2 +- .../hdfs/web/TestWebHdfsContentLength.java | 23 +- .../src/test/resources/log4j.properties | 49 ++ .../hadoop-hdfs-native-client/pom.xml | 46 +- .../src/CMakeLists.txt | 8 +- .../main/native/libhdfs-tests/CMakeLists.txt | 1 + .../src/main/native/libhdfs/CMakeLists.txt | 17 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 22 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 114 ++++- .../dev-support/findbugsExcludeFile.xml | 7 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - .../hadoop-hdfs/src/main/bin/hdfs | 10 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 + .../server/blockmanagement/BlockManager.java | 8 +- .../BlockPlacementPolicyDefault.java | 16 +- .../server/blockmanagement/DatanodeManager.java | 19 +- .../server/datanode/BlockRecoveryWorker.java | 69 ++- .../hdfs/server/datanode/BlockSender.java | 2 +- .../hadoop/hdfs/server/datanode/DataNode.java | 2 +- .../hdfs/server/datanode/DataStorage.java | 12 - .../datanode/fsdataset/impl/FsDatasetUtil.java | 5 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 4 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 3 +- .../hdfs/server/namenode/FSDirectory.java | 8 +- .../hdfs/server/namenode/FSNamesystem.java | 38 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 4 +- .../hadoop/hdfs/server/namenode/NameNode.java | 13 +- .../web/resources/NamenodeWebHdfsMethods.java | 8 + .../server/protocol/ReplicaRecoveryInfo.java | 6 + .../org/apache/hadoop/hdfs/web/JsonUtil.java | 15 + .../src/main/resources/hdfs-default.xml | 9 + .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 84 +++- .../apache/hadoop/fi/DataTransferTestUtil.java | 485 ------------------- .../test/aop/org/apache/hadoop/fi/FiConfig.java | 55 --- .../org/apache/hadoop/fi/FiHFlushTestUtil.java | 66 --- .../aop/org/apache/hadoop/fi/FiTestUtil.java | 209 -------- .../test/aop/org/apache/hadoop/fi/Pipeline.java | 50 -- .../aop/org/apache/hadoop/fi/PipelineTest.java | 27 -- .../org/apache/hadoop/fi/ProbabilityModel.java | 107 ---- .../org/apache/hadoop/fs/TestFiListPath.java | 105 ---- .../aop/org/apache/hadoop/fs/TestFiRename.java | 272 ----------- .../org/apache/hadoop/hdfs/DFSClientAspects.aj | 103 ---- .../aop/org/apache/hadoop/hdfs/HFlushAspects.aj | 64 --- .../apache/hadoop/hdfs/PipelinesTestUtil.java | 153 ------ .../org/apache/hadoop/hdfs/TestFiHFlush.java | 180 ------- .../org/apache/hadoop/hdfs/TestFiPipelines.java | 247 ---------- .../hdfs/protocol/ClientProtocolAspects.aj | 38 -- .../server/datanode/BlockReceiverAspects.aj | 232 --------- .../datanode/DataTransferProtocolAspects.aj | 80 --- .../hdfs/server/datanode/FSDatasetAspects.aj | 63 --- .../datanode/TestFiDataTransferProtocol.java | 312 ------------ .../datanode/TestFiDataTransferProtocol2.java | 289 ----------- .../server/datanode/TestFiPipelineClose.java | 250 ---------- .../server/namenode/FileDataServletAspects.aj | 46 -- .../hdfs/server/namenode/ListPathAspects.aj | 48 -- .../hdfs/server/namenode/RenameAspects.aj | 66 --- .../hdfs/ErasureCodeBenchmarkThroughput.java | 423 ++++++++++++++++ .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 2 + .../hadoop/hdfs/TestDFSClientFailover.java | 35 ++ .../org/apache/hadoop/hdfs/TestDFSShell.java | 69 +-- .../apache/hadoop/hdfs/TestDecommission.java | 35 ++ .../TestErasureCodeBenchmarkThroughput.java | 116 +++++ .../hadoop/hdfs/TestFsShellPermission.java | 12 +- .../java/org/apache/hadoop/hdfs/TestQuota.java | 74 ++- .../blockmanagement/TestHostFileManager.java | 2 +- .../server/blockmanagement/TestNodeCount.java | 24 +- .../TestReplicationPolicyConsiderLoad.java | 56 +++ .../datanode/FsDatasetImplTestUtilsFactory.java | 5 + .../server/datanode/FsDatasetTestUtils.java | 29 ++ .../hdfs/server/datanode/TestBlockRecovery.java | 37 ++ .../TestDataNodeMultipleRegistrations.java | 8 +- .../fsdataset/impl/FsDatasetImplTestUtils.java | 43 +- .../server/namenode/NNThroughputBenchmark.java | 42 +- .../hdfs/server/namenode/TestBackupNode.java | 71 +++ .../hdfs/server/namenode/TestClusterId.java | 36 +- .../TestCommitBlockSynchronization.java | 4 +- .../hdfs/server/namenode/TestFileTruncate.java | 35 +- .../hdfs/server/namenode/TestHDFSConcat.java | 13 + .../namenode/TestMetadataVersionOutput.java | 25 +- .../namenode/TestNamenodeCapacityReport.java | 14 +- .../hdfs/server/namenode/ha/TestHAMetrics.java | 56 +++ .../server/namenode/ha/TestQuotasWithHA.java | 15 + .../namenode/snapshot/TestSnapshotDeletion.java | 45 +- .../namenode/snapshot/TestSnapshotRename.java | 45 +- .../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 17 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 80 +++ .../org/apache/hadoop/tools/TestJMXGet.java | 17 +- .../java/org/apache/hadoop/tools/TestTools.java | 5 + .../apache/hadoop/tracing/TestTraceAdmin.java | 9 +- hadoop-mapreduce-project/CHANGES.txt | 29 +- hadoop-mapreduce-project/bin/mapred | 10 + .../hadoop/mapreduce/v2/app/MRAppMaster.java | 63 ++- .../v2/app/commit/CommitterEventHandler.java | 25 +- .../hadoop/mapred/FileOutputCommitter.java | 7 +- .../apache/hadoop/mapred/OutputCommitter.java | 34 +- .../hadoop/mapreduce/OutputCommitter.java | 26 + .../lib/output/FileOutputCommitter.java | 71 ++- .../hadoop/mapred/TestFileOutputCommitter.java | 151 ++++++ .../TestClientDistributedCacheManager.java | 40 +- .../lib/output/TestFileOutputCommitter.java | 197 +++++++- .../mapreduce/v2/hs/webapp/HsJobsBlock.java | 48 +- .../mapred/TestMRTimelineEventHandling.java | 10 + .../apache/hadoop/mapred/ShuffleHandler.java | 2 +- hadoop-project-dist/pom.xml | 13 +- hadoop-project/pom.xml | 1 + .../s3a/BlockingThreadPoolExecutorService.java | 272 +++++++++++ .../org/apache/hadoop/fs/s3a/Constants.java | 13 +- .../hadoop/fs/s3a/S3AFastOutputStream.java | 4 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 82 +--- .../src/site/markdown/tools/hadoop-aws/index.md | 10 +- .../TestBlockingThreadPoolExecutorService.java | 182 +++++++ .../fs/s3a/TestS3ABlockingThreadPool.java | 80 +++ .../org/apache/hadoop/tools/package-info.java | 4 +- .../src/main/native/pipes/impl/HadoopPipes.cc | 4 +- .../main/native/utils/api/hadoop/SerialUtils.hh | 1 + .../src/main/native/utils/impl/SerialUtils.cc | 7 + hadoop-yarn-project/CHANGES.txt | 47 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 10 + .../yarn/api/records/ApplicationAttemptId.java | 2 +- .../hadoop/yarn/api/records/ApplicationId.java | 2 +- .../distributedshell/ApplicationMaster.java | 20 +- .../hadoop/yarn/client/api/AMRMClient.java | 25 +- .../apache/hadoop/yarn/client/api/NMClient.java | 16 + .../yarn/client/api/async/AMRMClientAsync.java | 172 ++++++- .../yarn/client/api/async/NMClientAsync.java | 154 +++++- .../api/async/impl/AMRMClientAsyncImpl.java | 53 +- .../api/async/impl/NMClientAsyncImpl.java | 127 ++++- .../yarn/client/api/impl/AMRMClientImpl.java | 149 +++++- .../yarn/client/api/impl/NMClientImpl.java | 35 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 3 +- .../api/async/impl/TestAMRMClientAsync.java | 74 ++- .../api/async/impl/TestNMClientAsync.java | 93 +++- .../yarn/client/api/impl/TestAMRMClient.java | 161 +++++- .../api/impl/TestAMRMClientOnRMRestart.java | 80 ++- .../yarn/client/api/impl/TestNMClient.java | 31 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 32 ++ .../src/main/resources/yarn-default.xml | 8 +- .../ApplicationHistoryServer.java | 10 +- .../localizer/LocalResourcesTrackerImpl.java | 10 +- .../TestLocalResourcesTrackerImpl.java | 56 ++- .../TestResourceLocalizationService.java | 4 +- .../metrics/SystemMetricsPublisher.java | 4 +- .../ProportionalCapacityPreemptionPolicy.java | 3 +- .../recovery/FileSystemRMStateStore.java | 12 - .../recovery/LeveldbRMStateStore.java | 8 - .../recovery/MemoryRMStateStore.java | 17 - .../recovery/NullRMStateStore.java | 10 - .../resourcemanager/recovery/RMStateStore.java | 51 -- .../recovery/RMStateStoreEventType.java | 1 - .../recovery/ZKRMStateStore.java | 11 - .../server/resourcemanager/rmapp/RMAppImpl.java | 2 +- .../rmapp/attempt/RMAppAttemptImpl.java | 26 +- .../scheduler/AppSchedulingInfo.java | 4 +- .../resourcemanager/scheduler/QueueMetrics.java | 37 ++ .../CapacitySchedulerConfiguration.java | 14 +- .../scheduler/capacity/LeafQueue.java | 45 +- .../allocator/AbstractContainerAllocator.java | 2 +- .../allocator/RegularContainerAllocator.java | 17 +- .../webapp/CapacitySchedulerPage.java | 3 +- .../TestWorkPreservingRMRestart.java | 41 ++ .../metrics/TestSystemMetricsPublisher.java | 2 +- ...estProportionalCapacityPreemptionPolicy.java | 7 +- ...pacityPreemptionPolicyForNodePartitions.java | 6 + .../recovery/RMStateStoreTestBase.java | 5 +- .../scheduler/TestQueueMetrics.java | 55 +++ .../scheduler/capacity/TestLeafQueue.java | 426 +++++++++++++++- 237 files changed, 7002 insertions(+), 4766 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7a7fd5e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java ----------------------------------------------------------------------
