Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/48a8c9c3 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/48a8c9c3 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/48a8c9c3 Branch: refs/heads/HDFS-1312 Commit: 48a8c9c3e2cc3b24913a196dcf6b47ba93a07a92 Parents: 12b4cf7 500e5a5 Author: Anu Engineer <aengin...@apache.org> Authored: Tue Apr 5 12:28:48 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Apr 5 12:28:48 2016 -0700 ---------------------------------------------------------------------- BUILDING.txt | 6 +- dev-support/bin/dist-tools-hooks-maker | 182 +++ dev-support/docker/Dockerfile | 8 + .../server/AuthenticationFilter.java | 33 +- .../JWTRedirectAuthenticationHandler.java | 16 +- .../TestJWTRedirectAuthentictionHandler.java | 37 +- .../hadoop-common/src/main/bin/hadoop | 22 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 6 +- .../hadoop-common/src/main/bin/hadoop-daemon.sh | 6 +- .../src/main/bin/hadoop-daemons.sh | 6 +- .../src/main/bin/hadoop-functions.sh | 127 ++- .../src/main/bin/hadoop-layout.sh.example | 22 +- .../hadoop-common/src/main/bin/slaves.sh | 6 +- .../hadoop-common/src/main/bin/start-all.sh | 4 +- .../hadoop-common/src/main/bin/stop-all.sh | 4 +- .../hadoop-common/src/main/conf/hadoop-env.sh | 31 +- .../org/apache/hadoop/conf/Configuration.java | 13 + .../crypto/key/kms/KMSClientProvider.java | 8 +- .../hadoop/crypto/key/kms/ValueQueue.java | 16 +- .../hadoop/fs/CommonConfigurationKeys.java | 14 +- .../java/org/apache/hadoop/fs/FileSystem.java | 126 ++- .../java/org/apache/hadoop/fs/FileUtil.java | 41 - .../org/apache/hadoop/fs/PathIOException.java | 9 + .../hadoop/fs/shell/CommandWithDestination.java | 3 +- .../apache/hadoop/fs/shell/MoveCommands.java | 6 +- .../java/org/apache/hadoop/fs/shell/Touch.java | 3 +- .../apache/hadoop/io/erasurecode/CodecUtil.java | 13 +- .../io/erasurecode/rawcoder/RSRawDecoder.java | 175 +++ .../io/erasurecode/rawcoder/RSRawDecoder2.java | 176 --- .../io/erasurecode/rawcoder/RSRawEncoder.java | 75 ++ .../io/erasurecode/rawcoder/RSRawEncoder2.java | 76 -- .../rawcoder/RSRawErasureCoderFactory.java | 37 + .../rawcoder/RSRawErasureCoderFactory2.java | 37 - .../io/erasurecode/rawcoder/util/RSUtil.java | 149 ++- .../io/erasurecode/rawcoder/util/RSUtil2.java | 172 --- .../org/apache/hadoop/ipc/CallQueueManager.java | 134 ++- .../main/java/org/apache/hadoop/ipc/Client.java | 166 +-- .../apache/hadoop/ipc/DecayRpcScheduler.java | 396 +++++-- .../hadoop/ipc/DecayRpcSchedulerMXBean.java | 2 + .../apache/hadoop/ipc/DefaultRpcScheduler.java | 45 + .../org/apache/hadoop/ipc/FairCallQueue.java | 45 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 8 +- .../org/apache/hadoop/ipc/RpcScheduler.java | 8 +- .../java/org/apache/hadoop/ipc/Schedulable.java | 5 +- .../main/java/org/apache/hadoop/ipc/Server.java | 77 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 47 +- .../hadoop/metrics2/lib/MutableQuantiles.java | 7 +- .../hadoop/metrics2/util/QuantileEstimator.java | 32 + .../hadoop/metrics2/util/SampleQuantiles.java | 2 +- .../java/org/apache/hadoop/net/NetUtils.java | 31 +- .../org/apache/hadoop/net/NetworkTopology.java | 17 +- .../java/org/apache/hadoop/net/NodeBase.java | 18 +- .../apache/hadoop/security/SecurityUtil.java | 53 +- .../apache/hadoop/security/ssl/SSLFactory.java | 42 +- .../apache/hadoop/service/AbstractService.java | 2 +- .../org/apache/hadoop/tracing/TraceUtils.java | 4 +- .../hadoop/util/NativeLibraryChecker.java | 8 +- .../main/java/org/apache/hadoop/util/Shell.java | 10 +- .../apache/hadoop/util/ShutdownHookManager.java | 116 +- .../hadoop/io/compress/lz4/Lz4Compressor.c | 4 +- .../src/org/apache/hadoop/io/compress/lz4/lz4.c | 2 +- .../hadoop/io/erasurecode/erasure_coder.c | 1 + .../apache/hadoop/io/erasurecode/isal_load.c | 46 +- .../apache/hadoop/io/erasurecode/isal_load.h | 6 +- .../io/erasurecode/jni_erasure_code_native.c | 11 +- .../src/main/resources/core-default.xml | 11 +- .../src/site/markdown/ClusterSetup.md | 40 +- .../src/site/markdown/CommandsManual.md | 2 +- .../src/site/markdown/GroupsMapping.md | 26 +- .../src/site/markdown/HttpAuthentication.md | 6 +- .../src/site/markdown/UnixShellGuide.md | 2 +- .../hadoop/crypto/key/TestValueQueue.java | 6 - .../java/org/apache/hadoop/fs/TestFileUtil.java | 57 - .../org/apache/hadoop/fs/TestFsShellCopy.java | 52 +- .../org/apache/hadoop/fs/TestFsShellTouch.java | 88 ++ .../AbstractContractGetFileStatusTest.java | 23 + .../coder/TestHHXORErasureCoder.java | 4 +- .../erasurecode/coder/TestRSErasureCoder.java | 8 +- .../io/erasurecode/rawcoder/TestRSRawCoder.java | 33 + .../erasurecode/rawcoder/TestRSRawCoder2.java | 33 - .../apache/hadoop/ipc/TestCallQueueManager.java | 156 ++- .../hadoop/ipc/TestDecayRpcScheduler.java | 42 +- .../apache/hadoop/ipc/TestFairCallQueue.java | 79 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 101 +- .../hadoop/ipc/TestIPCServerResponder.java | 30 +- .../hadoop/ipc/TestIdentityProviders.java | 18 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 159 ++- .../java/org/apache/hadoop/ipc/TestRpcBase.java | 192 +++- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 397 ++----- .../org/apache/hadoop/net/TestNetUtils.java | 47 +- .../hadoop/security/TestDoAsEffectiveUser.java | 6 +- .../hadoop/security/TestSecurityUtil.java | 16 +- .../hadoop/security/ssl/TestSSLFactory.java | 139 ++- .../hadoop/util/TestShutdownHookManager.java | 57 +- .../hadoop-common/src/test/proto/test.proto | 9 + .../src/test/proto/test_rpc_service.proto | 4 + ...apache.hadoop.security.token.TokenIdentifier | 2 +- .../scripts/hadoop-functions_test_helper.bash | 3 +- .../scripts/hadoop_add_to_classpath_tools.bats | 47 + .../hadoop_add_to_classpath_toolspath.bats | 74 -- .../src/test/scripts/hadoop_basic_init.bats | 5 +- .../src/test/scripts/hadoop_bootstrap.bats | 19 +- .../src/test/scripts/hadoop_confdir.bats | 24 +- .../src/test/scripts/hadoop_entry_tests.bats | 49 + .../src/test/scripts/hadoop_finalize.bats | 2 +- .../src/test/scripts/hadoop_shellprofile.bats | 7 + .../hadoop-kms/src/main/conf/kms-env.sh | 4 +- .../hadoop/crypto/key/kms/server/KMS.java | 14 +- .../hadoop-kms/src/main/libexec/kms-config.sh | 8 +- .../hadoop-kms/src/main/sbin/kms.sh | 4 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 42 + hadoop-dist/pom.xml | 18 + .../dev-support/findbugsExcludeFile.xml | 1 + .../org/apache/hadoop/hdfs/BlockReader.java | 10 +- .../apache/hadoop/hdfs/BlockReaderFactory.java | 7 +- .../apache/hadoop/hdfs/BlockReaderLocal.java | 10 +- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 10 +- .../org/apache/hadoop/hdfs/ClientContext.java | 56 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 26 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 30 +- .../hadoop/hdfs/DFSStripedInputStream.java | 3 - .../org/apache/hadoop/hdfs/DFSUtilClient.java | 6 +- .../apache/hadoop/hdfs/ExternalBlockReader.java | 10 +- .../apache/hadoop/hdfs/FileChecksumHelper.java | 187 +++- .../apache/hadoop/hdfs/RemoteBlockReader.java | 29 +- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 29 +- .../org/apache/hadoop/hdfs/ReplicaAccessor.java | 7 + .../hadoop/hdfs/client/impl/DfsClientConf.java | 2 +- .../org/apache/hadoop/hdfs/protocol/Block.java | 4 +- .../hdfs/protocol/DatanodeAdminProperties.java | 100 ++ .../apache/hadoop/hdfs/protocol/DatanodeID.java | 6 + .../hadoop/hdfs/protocol/StripedBlockInfo.java | 61 + .../datatransfer/DataTransferProtocol.java | 16 +- .../hadoop/hdfs/protocol/datatransfer/Op.java | 1 + .../hdfs/protocol/datatransfer/Sender.java | 19 + .../hadoop/hdfs/protocolPB/PBHelperClient.java | 42 +- .../hdfs/util/CombinedHostsFileReader.java | 76 ++ .../hdfs/util/CombinedHostsFileWriter.java | 69 ++ .../hadoop/hdfs/util/StripedBlockUtil.java | 12 + .../src/main/proto/datatransfer.proto | 9 +- .../src/main/conf/httpfs-env.sh | 4 +- .../src/main/libexec/httpfs-config.sh | 8 +- .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 4 +- .../src/main/native/fuse-dfs/doc/README | 6 +- .../src/main/native/fuse-dfs/fuse_connect.c | 5 + .../main/native/fuse-dfs/fuse_dfs_wrapper.sh | 12 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 + .../src/main/bin/distribute-exclude.sh | 4 +- .../hadoop-hdfs/src/main/bin/hdfs | 9 +- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 6 +- .../src/main/bin/refresh-namenodes.sh | 4 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/start-dfs.sh | 4 +- .../src/main/bin/start-secure-dns.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-dfs.sh | 4 +- .../hadoop-hdfs/src/main/bin/stop-secure-dns.sh | 4 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 8 +- .../hdfs/protocol/datatransfer/Receiver.java | 28 + .../hadoop/hdfs/server/balancer/Balancer.java | 2 +- .../server/blockmanagement/BlockManager.java | 268 ++--- .../CombinedHostFileManager.java | 250 +++++ .../server/blockmanagement/DatanodeManager.java | 59 +- .../blockmanagement/DatanodeStorageInfo.java | 10 + .../blockmanagement/DecommissionManager.java | 30 +- .../blockmanagement/HostConfigManager.java | 80 ++ .../server/blockmanagement/HostFileManager.java | 147 +-- .../hdfs/server/blockmanagement/HostSet.java | 114 ++ .../blockmanagement/LowRedundancyBlocks.java | 458 ++++++++ .../blockmanagement/UnderReplicatedBlocks.java | 448 -------- .../hdfs/server/datanode/BPServiceActor.java | 5 + .../server/datanode/BlockChecksumHelper.java | 284 ++++- .../hdfs/server/datanode/BlockReceiver.java | 4 +- .../hdfs/server/datanode/BlockSender.java | 6 +- .../server/datanode/DataNodeFaultInjector.java | 2 + .../hdfs/server/datanode/DataXceiver.java | 43 + .../hdfs/server/datanode/DataXceiverServer.java | 2 +- .../datanode/IncrementalBlockReportManager.java | 9 + .../hdfs/server/datanode/ReplicaInPipeline.java | 53 +- .../erasurecode/ErasureCodingWorker.java | 6 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 158 ++- .../datanode/fsdataset/impl/FsVolumeImpl.java | 6 + .../datanode/fsdataset/impl/ReplicaMap.java | 2 +- .../server/namenode/EncryptionZoneManager.java | 15 + .../server/namenode/FSDirEncryptionZoneOp.java | 83 ++ .../hdfs/server/namenode/FSDirWriteFileOp.java | 4 + .../hdfs/server/namenode/FSDirectory.java | 48 +- .../hdfs/server/namenode/FSNamesystem.java | 27 + .../hadoop/hdfs/server/namenode/NameNode.java | 9 +- .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../OfflineImageReconstructor.java | 312 +++--- .../offlineImageViewer/PBImageXmlWriter.java | 463 +++++--- .../src/main/resources/hdfs-default.xml | 41 + .../src/main/webapps/hdfs/dfshealth.html | 13 +- .../src/main/webapps/hdfs/explorer.js | 33 +- .../hadoop-hdfs/src/site/markdown/Federation.md | 18 +- .../src/site/markdown/HDFSCommands.md | 159 ++- .../markdown/HDFSHighAvailabilityWithNFS.md | 4 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 4 +- .../src/site/markdown/HdfsNfsGateway.md | 8 +- .../src/site/markdown/HdfsUserGuide.md | 6 +- .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 61 +- .../org/apache/hadoop/TestRefreshCallQueue.java | 2 +- .../hadoop/fs/TestEnhancedByteBufferAccess.java | 4 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +- .../hadoop/hdfs/TestBlockReaderLocal.java | 4 +- .../hadoop/hdfs/TestBlocksScheduledCounter.java | 51 + .../TestClientProtocolForPipelineRecovery.java | 53 + .../org/apache/hadoop/hdfs/TestConnCache.java | 2 - .../org/apache/hadoop/hdfs/TestDFSShell.java | 31 + .../apache/hadoop/hdfs/TestDatanodeReport.java | 57 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 62 ++ .../apache/hadoop/hdfs/TestEncryptionZones.java | 4 +- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 36 +- .../hadoop/hdfs/TestExternalBlockReader.java | 8 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 247 +++++ .../datatransfer/sasl/TestSaslDataTransfer.java | 48 + .../blockmanagement/BlockManagerTestUtil.java | 2 +- .../blockmanagement/TestBlockManager.java | 20 +- .../TestBlocksWithNotEnoughRacks.java | 34 +- .../blockmanagement/TestDatanodeManager.java | 8 +- .../blockmanagement/TestHostFileManager.java | 10 +- .../TestLowRedundancyBlockQueues.java | 182 +++ .../blockmanagement/TestPendingReplication.java | 14 +- .../blockmanagement/TestReplicationPolicy.java | 158 +-- .../TestUnderReplicatedBlockQueues.java | 182 --- .../server/datanode/TestBPOfferService.java | 96 +- .../hdfs/server/datanode/TestBlockRecovery.java | 137 ++- .../hdfs/server/datanode/TestDataNodeUUID.java | 52 + .../server/datanode/TestFsDatasetCache.java | 5 +- .../fsdataset/impl/TestFsDatasetImpl.java | 66 ++ .../namenode/TestDecommissioningStatus.java | 12 +- .../hdfs/server/namenode/TestHostsFiles.java | 70 +- .../hdfs/server/namenode/TestMetaSave.java | 2 +- .../server/namenode/TestNameNodeMXBean.java | 25 +- .../namenode/TestProtectedDirectories.java | 44 + .../hdfs/server/namenode/TestStartup.java | 54 +- .../TestUpgradeDomainBlockPlacementPolicy.java | 169 +++ .../namenode/ha/TestStandbyCheckpoints.java | 5 + .../apache/hadoop/hdfs/tools/TestDFSAdmin.java | 2 +- .../hadoop/hdfs/util/HostsFileWriter.java | 122 ++ .../hdfs/util/TestCombinedHostsFileReader.java | 79 ++ .../apache/hadoop/net/TestNetworkTopology.java | 7 + .../apache/hadoop/tracing/TestTraceAdmin.java | 2 +- .../TestTracingShortCircuitLocalRead.java | 4 +- .../src/test/resources/dfs.hosts.json | 5 + .../src/test/resources/testHDFSConf.xml | 6 +- hadoop-mapreduce-project/bin/mapred | 14 +- hadoop-mapreduce-project/bin/mapred-config.sh | 6 +- .../bin/mr-jobhistory-daemon.sh | 4 +- .../org/apache/hadoop/mapred/ClusterStatus.java | 35 +- .../apache/hadoop/mapred/pipes/Submitter.java | 2 +- .../org/apache/hadoop/mapreduce/TaskID.java | 57 +- .../org/apache/hadoop/mapreduce/tools/CLI.java | 48 +- .../hadoop/mapreduce/util/ProcessTree.java | 7 +- .../src/site/markdown/EncryptedShuffle.md | 2 +- .../org/apache/hadoop/mapreduce/TestTaskID.java | 461 ++++++++ .../apache/hadoop/mapreduce/tools/TestCLI.java | 38 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 5 +- .../org/apache/hadoop/mapred/NotRunningJob.java | 7 +- .../java/org/apache/hadoop/fs/DFSCIOTest.java | 10 +- .../org/apache/hadoop/fs/TestFileSystem.java | 20 +- .../java/org/apache/hadoop/fs/TestJHLA.java | 6 +- .../java/org/apache/hadoop/hdfs/NNBench.java | 240 ++-- .../org/apache/hadoop/hdfs/TestNNBench.java | 84 ++ .../io/TestSequenceFileMergeProgress.java | 12 +- .../hadoop/mapred/ClusterMapReduceTestCase.java | 14 +- .../apache/hadoop/mapred/ReliabilityTest.java | 2 +- .../apache/hadoop/mapred/TestAuditLogger.java | 9 +- .../apache/hadoop/mapred/TestBadRecords.java | 8 +- .../mapred/TestClusterMapReduceTestCase.java | 10 + .../org/apache/hadoop/mapred/TestCollect.java | 10 +- .../mapred/TestCommandLineJobSubmission.java | 9 +- .../hadoop/mapred/TestFieldSelection.java | 11 +- .../mapred/TestFileInputFormatPathFilter.java | 19 +- .../apache/hadoop/mapred/TestGetSplitHosts.java | 7 +- .../apache/hadoop/mapred/TestIFileStreams.java | 13 +- .../org/apache/hadoop/mapred/TestInputPath.java | 7 +- .../hadoop/mapred/TestJavaSerialization.java | 10 +- .../org/apache/hadoop/mapred/TestJobName.java | 6 + .../hadoop/mapred/TestJobSysDirWithDFS.java | 10 +- .../mapred/TestKeyValueTextInputFormat.java | 15 +- .../apache/hadoop/mapred/TestLazyOutput.java | 7 +- .../hadoop/mapred/TestMRCJCFileInputFormat.java | 32 +- .../mapred/TestMRCJCFileOutputCommitter.java | 28 +- .../mapred/TestMRTimelineEventHandling.java | 19 +- .../apache/hadoop/mapred/TestMapProgress.java | 9 +- .../org/apache/hadoop/mapred/TestMerge.java | 7 +- .../apache/hadoop/mapred/TestMiniMRBringup.java | 6 +- .../hadoop/mapred/TestMiniMRChildTask.java | 16 +- .../hadoop/mapred/TestMiniMRDFSCaching.java | 14 +- .../hadoop/mapred/TestMultiFileInputFormat.java | 19 +- .../hadoop/mapred/TestMultiFileSplit.java | 10 +- .../hadoop/mapred/TestMultipleLevelCaching.java | 12 +- .../mapred/TestMultipleTextOutputFormat.java | 23 +- .../apache/hadoop/mapred/TestNetworkedJob.java | 89 +- .../hadoop/mapred/TestOldCombinerGrouping.java | 5 +- .../apache/hadoop/mapred/TestReduceFetch.java | 10 +- .../mapred/TestReduceFetchFromPartialMem.java | 46 +- .../apache/hadoop/mapred/TestReduceTask.java | 18 +- .../TestSequenceFileAsBinaryInputFormat.java | 19 +- .../TestSequenceFileAsBinaryOutputFormat.java | 31 +- .../TestSequenceFileAsTextInputFormat.java | 33 +- .../mapred/TestSequenceFileInputFilter.java | 32 +- .../mapred/TestSequenceFileInputFormat.java | 29 +- .../apache/hadoop/mapred/TestSortedRanges.java | 19 +- .../TestSpecialCharactersInOutputPath.java | 21 +- .../hadoop/mapred/TestStatisticsCollector.java | 10 +- .../hadoop/mapred/TestUserDefinedCounters.java | 24 +- .../hadoop/mapred/TestWritableJobConf.java | 20 +- .../apache/hadoop/mapred/TestYARNRunner.java | 8 +- .../hadoop/mapred/join/TestDatamerge.java | 42 +- .../hadoop/mapred/join/TestTupleWritable.java | 24 +- .../TestWrappedRecordReaderClassloader.java | 7 +- .../mapred/lib/TestDelegatingInputFormat.java | 9 +- .../hadoop/mapred/lib/TestLineInputFormat.java | 7 +- .../hadoop/mapred/lib/TestMultipleInputs.java | 2 - .../mapred/lib/aggregate/TestAggregates.java | 14 +- .../mapred/lib/db/TestConstructQuery.java | 16 +- .../apache/hadoop/mapred/pipes/TestPipes.java | 9 +- .../hadoop/mapreduce/TestLocalRunner.java | 34 +- .../hadoop/mapreduce/TestMRJobClient.java | 49 +- .../mapreduce/TestMapReduceLazyOutput.java | 9 +- .../mapreduce/TestNewCombinerGrouping.java | 5 +- .../hadoop/mapreduce/TestValueIterReset.java | 8 +- .../TestYarnClientProtocolProvider.java | 5 +- .../lib/aggregate/TestMapReduceAggregates.java | 28 +- .../mapreduce/lib/db/TestDBOutputFormat.java | 17 +- .../mapreduce/lib/db/TestIntegerSplitter.java | 15 +- .../mapreduce/lib/db/TestTextSplitter.java | 18 +- .../lib/fieldsel/TestMRFieldSelection.java | 20 +- .../lib/input/TestCombineFileInputFormat.java | 4 +- .../TestMRSequenceFileAsBinaryInputFormat.java | 21 +- .../TestMRSequenceFileAsTextInputFormat.java | 27 +- .../input/TestMRSequenceFileInputFilter.java | 39 +- .../lib/input/TestNLineInputFormat.java | 34 +- .../mapreduce/lib/join/TestJoinDatamerge.java | 52 +- .../mapreduce/lib/join/TestJoinProperties.java | 44 +- .../lib/join/TestJoinTupleWritable.java | 24 +- .../lib/join/TestWrappedRRClassloader.java | 17 +- .../TestMRSequenceFileAsBinaryOutputFormat.java | 35 +- .../lib/partition/TestBinaryPartitioner.java | 16 +- .../lib/partition/TestKeyFieldHelper.java | 9 +- .../TestMRKeyFieldBasedPartitioner.java | 6 +- .../partition/TestTotalOrderPartitioner.java | 11 +- .../mapreduce/util/TestMRAsyncDiskService.java | 15 +- .../mapreduce/v2/TestMRJobsWithProfiler.java | 2 +- .../mapreduce/v2/TestMiniMRProxyUser.java | 30 +- .../hadoop/mapreduce/v2/TestNonExistentJob.java | 18 +- .../apache/hadoop/examples/TestWordStats.java | 15 +- hadoop-tools/hadoop-archive-logs/pom.xml | 17 + .../apache/hadoop/tools/HadoopArchiveLogs.java | 4 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 4 +- hadoop-tools/hadoop-archives/pom.xml | 17 + hadoop-tools/hadoop-aws/pom.xml | 17 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 77 +- .../src/site/markdown/tools/hadoop-aws/index.md | 12 +- .../s3a/TestS3AContractGetFileStatus.java | 31 + .../src/test/resources/contract/s3a.xml | 5 + hadoop-tools/hadoop-azure/pom.xml | 18 + .../hadoop-azure/src/site/markdown/index.md | 3 + hadoop-tools/hadoop-datajoin/pom.xml | 16 + .../apache/hadoop/contrib/utils/join/README.txt | 2 +- hadoop-tools/hadoop-distcp/pom.xml | 11 + .../java/org/apache/hadoop/tools/DistCp.java | 79 -- .../apache/hadoop/tools/DistCpConstants.java | 16 - .../apache/hadoop/tools/DistCpOptionSwitch.java | 9 - .../org/apache/hadoop/tools/DistCpOptions.java | 22 - .../org/apache/hadoop/tools/OptionsParser.java | 5 - .../apache/hadoop/tools/mapred/CopyMapper.java | 68 -- .../src/site/markdown/DistCp.md.vm | 39 - .../apache/hadoop/tools/TestOptionsParser.java | 17 +- hadoop-tools/hadoop-extras/pom.xml | 17 + hadoop-tools/hadoop-gridmix/pom.xml | 17 + hadoop-tools/hadoop-kafka/pom.xml | 17 + hadoop-tools/hadoop-openstack/pom.xml | 17 + .../hadoop-openstack/src/site/markdown/index.md | 7 +- .../native/pipes/debug/pipes-default-script | 5 +- hadoop-tools/hadoop-rumen/pom.xml | 17 + hadoop-tools/hadoop-sls/pom.xml | 17 + .../hadoop-sls/src/main/bin/rumen2sls.sh | 6 +- hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 10 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 + .../yarn/sls/scheduler/RMNodeWrapper.java | 9 + hadoop-tools/hadoop-streaming/pom.xml | 17 + .../apache/hadoop/streaming/DumpTypedBytes.java | 2 +- .../hadoop/streaming/HadoopStreaming.java | 2 +- .../apache/hadoop/streaming/LoadTypedBytes.java | 2 +- .../org/apache/hadoop/streaming/StreamJob.java | 16 +- .../streaming/TestStreamingBadRecords.java | 9 +- .../hadoop-yarn/bin/start-yarn.sh | 4 +- .../hadoop-yarn/bin/stop-yarn.sh | 4 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 8 +- .../hadoop-yarn/bin/yarn-config.sh | 6 +- .../hadoop-yarn/bin/yarn-daemon.sh | 4 +- .../hadoop-yarn/bin/yarn-daemons.sh | 4 +- .../yarn/api/records/ResourceUtilization.java | 22 + .../hadoop/yarn/conf/YarnConfiguration.java | 15 + .../distributedshell/ApplicationMaster.java | 13 +- .../hadoop/yarn/client/ProtocolHATestBase.java | 3 +- .../hadoop/yarn/client/TestRMFailover.java | 9 +- .../yarn/client/api/impl/TestAMRMProxy.java | 413 +++++++ .../yarn/client/api/impl/TestYarnClient.java | 18 + .../api/impl/FileSystemTimelineWriter.java | 112 +- .../apache/hadoop/yarn/util/ConverterUtils.java | 16 +- .../yarn/util/ProcfsBasedProcessTree.java | 6 +- .../util/ResourceCalculatorProcessTree.java | 8 +- .../apache/hadoop/yarn/util/SystemClock.java | 5 +- .../yarn/util/WindowsBasedProcessTree.java | 5 +- .../hadoop/yarn/util/resource/Resources.java | 2 +- .../hadoop/yarn/webapp/util/WebAppUtils.java | 22 + .../src/main/resources/yarn-default.xml | 26 +- .../api/records/TestResourceUtilization.java | 63 ++ .../yarn/util/TestProcfsBasedProcessTree.java | 8 - .../util/TestResourceCalculatorProcessTree.java | 3 +- .../yarn/util/TestWindowsBasedProcessTree.java | 3 +- .../ApplicationHistoryServer.java | 15 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 6 + .../hadoop/yarn/server/webapp/WebServices.java | 22 +- .../hadoop-yarn-server-nodemanager/pom.xml | 5 + .../nodemanager/NodeStatusUpdaterImpl.java | 50 +- .../nodemanager/amrmproxy/AMRMProxyService.java | 14 +- .../amrmproxy/DefaultRequestInterceptor.java | 7 + .../containermanager/ContainerManagerImpl.java | 49 +- .../logaggregation/AppLogAggregatorImpl.java | 14 +- .../monitor/ContainerMetrics.java | 69 ++ .../nodemanager/webapp/NMWebServices.java | 6 +- .../TestDockerContainerExecutorWithMocks.java | 2 +- .../containermanager/TestNMProxy.java | 3 +- .../TestResourceLocalizationService.java | 2 +- .../TestLogAggregationService.java | 12 +- .../monitor/TestContainerMetrics.java | 58 +- .../webapp/TestNMWebServicesApps.java | 9 +- .../server/resourcemanager/AdminService.java | 40 +- .../server/resourcemanager/ClientRMService.java | 9 +- .../resourcemanager/NodesListManager.java | 104 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../resourcemanager/ResourceTrackerService.java | 88 +- .../monitor/SchedulingEditPolicy.java | 8 +- .../monitor/SchedulingMonitor.java | 4 - .../CapacitySchedulerPreemptionContext.java | 52 + .../CapacitySchedulerPreemptionUtils.java | 65 ++ .../capacity/FifoCandidatesSelector.java | 364 ++++++ .../capacity/PreemptableResourceCalculator.java | 370 +++++++ .../capacity/PreemptionCandidatesSelector.java | 52 + .../ProportionalCapacityPreemptionPolicy.java | 1044 ++++-------------- .../monitor/capacity/TempQueuePerPartition.java | 159 +++ .../resource/DynamicResourceConfiguration.java | 13 +- .../rmcontainer/RMContainer.java | 1 + .../server/resourcemanager/rmnode/RMNode.java | 4 + .../resourcemanager/rmnode/RMNodeImpl.java | 22 +- .../scheduler/AbstractYarnScheduler.java | 170 +-- .../scheduler/ClusterNodeTracker.java | 300 +++++ .../resourcemanager/scheduler/NodeFilter.java | 33 + .../scheduler/PreemptableResourceScheduler.java | 2 +- .../server/resourcemanager/scheduler/Queue.java | 20 + .../scheduler/ResourceLimits.java | 9 + .../scheduler/SchedulerNode.java | 9 +- .../scheduler/capacity/AbstractCSQueue.java | 71 +- .../scheduler/capacity/CSAssignment.java | 11 + .../scheduler/capacity/CSQueueUtils.java | 13 + .../scheduler/capacity/CapacityScheduler.java | 216 ++-- .../CapacitySchedulerConfiguration.java | 61 +- .../capacity/CapacitySchedulerContext.java | 15 +- .../scheduler/capacity/LeafQueue.java | 85 +- .../scheduler/capacity/ParentQueue.java | 157 ++- .../scheduler/capacity/QueueCapacities.java | 38 +- .../allocator/AbstractContainerAllocator.java | 2 + .../capacity/allocator/ContainerAllocation.java | 12 + .../allocator/RegularContainerAllocator.java | 39 +- .../capacity/preemption/KillableContainer.java | 45 + .../capacity/preemption/PreemptableQueue.java | 96 ++ .../capacity/preemption/PreemptionManager.java | 165 +++ .../scheduler/common/AssignmentInformation.java | 6 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 32 +- .../common/fica/FiCaSchedulerNode.java | 65 +- .../scheduler/event/SchedulerEventType.java | 15 +- .../scheduler/fair/FSAppAttempt.java | 13 +- .../resourcemanager/scheduler/fair/FSQueue.java | 8 + .../scheduler/fair/FairScheduler.java | 122 +- .../scheduler/fifo/FifoScheduler.java | 44 +- .../resourcemanager/webapp/AppAttemptPage.java | 4 +- .../webapp/CapacitySchedulerPage.java | 16 +- .../resourcemanager/webapp/NodesPage.java | 53 +- .../webapp/RMAppAttemptBlock.java | 61 +- .../resourcemanager/webapp/RMWebAppFilter.java | 40 +- .../resourcemanager/webapp/RMWebServices.java | 32 +- .../webapp/dao/CapacitySchedulerInfo.java | 26 +- .../webapp/dao/ReservationIdInfo.java | 64 -- .../webapp/dao/ReservationInfo.java | 7 +- .../yarn/server/resourcemanager/MockNodes.java | 9 + .../yarn/server/resourcemanager/MockRM.java | 7 +- .../ParameterizedSchedulerTestBase.java | 36 +- .../yarn/server/resourcemanager/TestRM.java | 4 - .../resourcemanager/TestRMAdminService.java | 77 +- .../resourcemanager/TestRMDispatcher.java | 4 +- .../server/resourcemanager/TestRMRestart.java | 8 +- .../TestResourceTrackerService.java | 222 +++- .../TestWorkPreservingRMRestart.java | 9 - .../applicationsmanager/TestAMRestart.java | 7 +- ...estProportionalCapacityPreemptionPolicy.java | 137 +-- ...pacityPreemptionPolicyForNodePartitions.java | 80 +- .../recovery/TestZKRMStateStorePerf.java | 9 +- .../reservation/TestReservationSystem.java | 4 - .../rmapp/TestNodesListManager.java | 5 +- .../scheduler/TestAbstractYarnScheduler.java | 20 +- .../capacity/TestApplicationLimits.java | 2 + .../capacity/TestApplicationPriority.java | 6 +- .../capacity/TestCapacityScheduler.java | 8 +- .../TestCapacitySchedulerPreemption.java | 683 ++++++++++++ .../scheduler/capacity/TestChildQueueOrder.java | 2 + .../capacity/TestContainerAllocation.java | 13 +- .../scheduler/capacity/TestLeafQueue.java | 3 + .../TestNodeLabelContainerAllocation.java | 176 ++- .../scheduler/capacity/TestParentQueue.java | 2 + .../scheduler/capacity/TestQueueCapacities.java | 4 +- .../scheduler/capacity/TestReservations.java | 21 +- .../scheduler/capacity/TestUtils.java | 36 + .../scheduler/fair/FairSchedulerTestBase.java | 22 +- .../fair/TestContinuousScheduling.java | 78 +- .../scheduler/fair/TestFairScheduler.java | 82 +- .../fair/TestFairSchedulerPreemption.java | 2 +- .../scheduler/fifo/TestFifoScheduler.java | 19 +- .../security/TestClientToAMTokens.java | 4 - .../security/TestDelegationTokenRenewer.java | 3 - .../resourcemanager/webapp/TestNodesPage.java | 37 +- .../webapp/TestRMWebServicesApps.java | 24 +- .../TestRMWebServicesAppsModification.java | 10 +- .../webapp/TestRMWebServicesCapacitySched.java | 3 + .../webapp/TestRMWebServicesNodes.java | 12 +- .../webapp/TestRMWebServicesReservation.java | 15 +- .../src/test/resources/test-fair-scheduler.xml | 34 +- .../hadoop/yarn/server/MiniYARNCluster.java | 144 ++- .../yarn/server/TestMiniYARNClusterForHA.java | 4 - .../src/site/markdown/CapacityScheduler.md | 22 +- .../site/markdown/DockerContainerExecutor.md.vm | 2 +- .../src/site/markdown/FairScheduler.md | 6 +- .../src/site/markdown/ResourceManagerRest.md | 217 ++++ .../src/site/markdown/SecureContainer.md | 2 +- .../src/site/markdown/TimelineServer.md | 8 +- .../src/site/markdown/YarnCommands.md | 45 +- 541 files changed, 16565 insertions(+), 6900 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/48a8c9c3/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/48a8c9c3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ----------------------------------------------------------------------
