Merge remote-tracking branch 'apache/trunk' into HDFS-7285
Conflicts:
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Change-Id: Ic7946c4ea35bed587fe879ce58b959b25ecc0823
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fd55202
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fd55202
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fd55202
Branch: refs/heads/HDFS-7240
Commit: 8fd55202468b28422b0df888641c9b08906fe4a7
Parents: 6419900 151fca5
Author: Zhe Zhang <zhezh...@cloudera.com>
Authored: Tue Sep 29 01:39:16 2015 -0700
Committer: Zhe Zhang <zhezh...@cloudera.com>
Committed: Tue Sep 29 01:39:16 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 29 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 33 +-
hadoop-common-project/hadoop-common/pom.xml | 45 +-
.../hadoop/fs/CommonConfigurationKeys.java | 3 +
.../fs/CommonConfigurationKeysPublic.java | 6 +
.../org/apache/hadoop/fs/FSOutputSummer.java | 12 +-
.../java/org/apache/hadoop/fs/FileContext.java | 7 +
.../java/org/apache/hadoop/fs/FileSystem.java | 27 +-
.../org/apache/hadoop/fs/FilterFileSystem.java | 1 -
.../main/java/org/apache/hadoop/fs/FsShell.java | 26 +-
.../java/org/apache/hadoop/fs/FsTracer.java | 64 +
.../main/java/org/apache/hadoop/fs/Globber.java | 15 +-
.../org/apache/hadoop/fs/HarFileSystem.java | 1 -
.../org/apache/hadoop/fs/LocalDirAllocator.java | 4 +-
.../apache/hadoop/fs/RawLocalFileSystem.java | 1 -
.../hadoop/fs/shell/CommandWithDestination.java | 8 +-
.../hadoop/fs/viewfs/ChRootedFileSystem.java | 1 -
.../main/java/org/apache/hadoop/ipc/Client.java | 13 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 17 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 51 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 13 +-
.../main/java/org/apache/hadoop/net/DNS.java | 129 +-
.../apache/hadoop/security/SecurityUtil.java | 37 +-
.../apache/hadoop/tracing/SpanReceiverHost.java | 208 --
.../org/apache/hadoop/tracing/TraceUtils.java | 24 +-
.../tracing/TracerConfigurationManager.java | 100 +
.../java/org/apache/hadoop/util/ProtoUtil.java | 13 +-
.../src/main/proto/RpcHeader.proto | 5 +-
.../src/main/resources/core-default.xml | 40 +
.../hadoop-common/src/site/markdown/Metrics.md | 1 -
.../hadoop-common/src/site/markdown/Tracing.md | 80 +-
.../apache/hadoop/conf/TestConfiguration.java | 2 +-
.../java/org/apache/hadoop/fs/TestFsShell.java | 44 +-
.../org/apache/hadoop/fs/TestHarFileSystem.java | 1 -
.../apache/hadoop/fs/TestLocalDirAllocator.java | 26 +-
.../apache/hadoop/fs/TestLocalFileSystem.java | 1 +
.../AbstractContractRootDirectoryTest.java | 14 +
.../apache/hadoop/ha/ClientBaseWithFixes.java | 40 +-
.../http/TestAuthenticationSessionCookie.java | 5 +-
.../apache/hadoop/http/TestHttpCookieFlag.java | 7 +-
.../apache/hadoop/http/TestSSLHttpServer.java | 4 +-
.../java/org/apache/hadoop/ipc/TestRPC.java | 4 +-
.../org/apache/hadoop/net/ServerSocketUtil.java | 7 +-
.../java/org/apache/hadoop/net/TestDNS.java | 110 +-
.../hadoop/security/TestSecurityUtil.java | 2 +-
.../hadoop/security/ssl/KeyStoreTestUtil.java | 60 +-
.../apache/hadoop/tracing/SetSpanReceiver.java | 13 +-
.../apache/hadoop/tracing/TestTraceUtils.java | 2 +-
.../dev-support/findbugsExcludeFile.xml | 24 +
.../org/apache/hadoop/fs/HdfsBlockLocation.java | 47 +
.../hadoop/hdfs/BlockMissingException.java | 65 +
.../apache/hadoop/hdfs/BlockReaderFactory.java | 906 +++++
.../apache/hadoop/hdfs/BlockReaderLocal.java | 21 +-
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 30 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3240 +++++++++++++++++
.../hadoop/hdfs/DFSClientFaultInjector.java | 60 +
.../hadoop/hdfs/DFSHedgedReadMetrics.java | 58 +
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 234 ++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 1889 ++++++++++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 982 ++++++
.../java/org/apache/hadoop/hdfs/DFSPacket.java | 364 ++
.../hadoop/hdfs/DFSStripedInputStream.java | 972 ++++++
.../hadoop/hdfs/DFSStripedOutputStream.java | 953 +++++
.../org/apache/hadoop/hdfs/DFSUtilClient.java | 32 +
.../org/apache/hadoop/hdfs/DataStreamer.java | 1944 +++++++++++
.../hadoop/hdfs/DistributedFileSystem.java | 2335 +++++++++++++
.../apache/hadoop/hdfs/ExternalBlockReader.java | 20 +-
.../hadoop/hdfs/HdfsConfigurationLoader.java | 44 +
.../hadoop/hdfs/NameNodeProxiesClient.java | 366 ++
.../apache/hadoop/hdfs/RemoteBlockReader.java | 20 +-
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 23 +-
.../apache/hadoop/hdfs/RemotePeerFactory.java | 43 +
.../org/apache/hadoop/hdfs/ReplicaAccessor.java | 10 +-
.../hadoop/hdfs/ReplicaAccessorBuilder.java | 3 +
.../apache/hadoop/hdfs/StripedDataStreamer.java | 187 +
.../hdfs/UnknownCipherSuiteException.java | 35 +
.../UnknownCryptoProtocolVersionException.java | 38 +
.../org/apache/hadoop/hdfs/XAttrHelper.java | 174 +
.../hdfs/client/HdfsClientConfigKeys.java | 11 +
.../hadoop/hdfs/client/HdfsDataInputStream.java | 113 +
.../hdfs/client/HdfsDataOutputStream.java | 112 +
.../client/impl/CorruptFileBlockIterator.java | 105 +
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 524 +++
.../hdfs/inotify/MissingEventsException.java | 54 +
.../hadoop/hdfs/protocol/AclException.java | 39 +
.../hdfs/protocol/CacheDirectiveIterator.java | 129 +
.../hadoop/hdfs/protocol/CachePoolIterator.java | 62 +
.../hdfs/protocol/EncryptionZoneIterator.java | 62 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 23 +
.../QuotaByStorageTypeExceededException.java | 56 +
.../hdfs/protocol/UnresolvedPathException.java | 87 +
.../datatransfer/DataTransferProtoUtil.java | 43 +-
.../datatransfer/ReplaceDatanodeOnFailure.java | 200 ++
.../hdfs/protocol/datatransfer/Sender.java | 22 +-
.../protocolPB/ClientNamenodeProtocolPB.java | 46 +
.../ClientNamenodeProtocolTranslatorPB.java | 1593 +++++++++
.../datanode/ReplicaNotFoundException.java | 53 +
.../namenode/RetryStartFileException.java | 36 +
.../ha/AbstractNNFailoverProxyProvider.java | 55 +
.../ha/WrappedFailoverProxyProvider.java | 71 +
.../hadoop/hdfs/util/StripedBlockUtil.java | 952 +++++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 1 -
.../src/main/proto/hdfs.proto | 178 +-
.../hadoop/hdfs/nfs/nfs3/OffsetRange.java | 4 +
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 141 +-
.../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 82 +-
.../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 92 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 80 +-
.../dev-support/findbugsExcludeFile.xml | 19 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 3 +-
.../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 1 +
.../bkjournal/src/main/proto/bkjournal.proto | 1 +
.../org/apache/hadoop/fs/HdfsBlockLocation.java | 47 -
.../hadoop/hdfs/BlockMissingException.java | 65 -
.../apache/hadoop/hdfs/BlockReaderFactory.java | 891 -----
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3244 ------------------
.../hadoop/hdfs/DFSClientFaultInjector.java | 57 -
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 39 +-
.../hadoop/hdfs/DFSHedgedReadMetrics.java | 58 -
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 239 --
.../org/apache/hadoop/hdfs/DFSInputStream.java | 1887 ----------
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 976 ------
.../java/org/apache/hadoop/hdfs/DFSPacket.java | 359 --
.../hadoop/hdfs/DFSStripedInputStream.java | 973 ------
.../hadoop/hdfs/DFSStripedOutputStream.java | 954 -----
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 31 -
.../org/apache/hadoop/hdfs/DataStreamer.java | 1900 ----------
.../hadoop/hdfs/DistributedFileSystem.java | 2338 -------------
.../java/org/apache/hadoop/hdfs/HAUtil.java | 9 +-
.../apache/hadoop/hdfs/HdfsConfiguration.java | 11 +-
.../org/apache/hadoop/hdfs/NameNodeProxies.java | 275 +-
.../apache/hadoop/hdfs/RemotePeerFactory.java | 43 -
.../apache/hadoop/hdfs/StripedDataStreamer.java | 187 -
.../hdfs/UnknownCipherSuiteException.java | 35 -
.../UnknownCryptoProtocolVersionException.java | 38 -
.../org/apache/hadoop/hdfs/XAttrHelper.java | 174 -
.../hadoop/hdfs/client/HdfsDataInputStream.java | 113 -
.../hdfs/client/HdfsDataOutputStream.java | 112 -
.../client/impl/CorruptFileBlockIterator.java | 105 -
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 524 ---
.../hdfs/inotify/MissingEventsException.java | 54 -
.../hadoop/hdfs/protocol/AclException.java | 39 -
.../hdfs/protocol/CacheDirectiveIterator.java | 130 -
.../hadoop/hdfs/protocol/CachePoolIterator.java | 63 -
.../hdfs/protocol/EncryptionZoneIterator.java | 64 -
.../QuotaByStorageTypeExceededException.java | 56 -
.../hdfs/protocol/UnresolvedPathException.java | 87 -
.../hdfs/protocol/datatransfer/Receiver.java | 35 +-
.../datatransfer/ReplaceDatanodeOnFailure.java | 200 --
.../protocolPB/ClientNamenodeProtocolPB.java | 46 -
.../ClientNamenodeProtocolTranslatorPB.java | 1593 ---------
.../DatanodeProtocolClientSideTranslatorPB.java | 2 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 4 +-
.../NamenodeProtocolServerSideTranslatorPB.java | 4 +-
.../NamenodeProtocolTranslatorPB.java | 5 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 33 +-
.../hdfs/qjournal/server/JournalNode.java | 13 +-
.../qjournal/server/JournalNodeRpcServer.java | 1 +
.../hadoop/hdfs/server/balancer/Dispatcher.java | 3 +-
.../server/blockmanagement/DatanodeManager.java | 45 +-
.../blockmanagement/HeartbeatManager.java | 30 +-
.../hdfs/server/common/HdfsServerConstants.java | 27 +-
.../hdfs/server/datanode/BlockReceiver.java | 4 +-
.../hdfs/server/datanode/BlockSender.java | 13 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 73 +-
.../hdfs/server/datanode/DataXceiver.java | 8 +-
.../hdfs/server/datanode/DirectoryScanner.java | 337 +-
.../datanode/ReplicaNotFoundException.java | 53 -
.../erasurecode/ErasureCodingWorker.java | 6 +-
.../datanode/fsdataset/impl/BlockPoolSlice.java | 4 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 8 +-
.../impl/RamDiskAsyncLazyPersistService.java | 4 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 3 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 2 +-
.../server/namenode/FSDirEncryptionZoneOp.java | 303 ++
.../server/namenode/FSDirStatAndListingOp.java | 27 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 6 +-
.../hdfs/server/namenode/FSDirectory.java | 147 +-
.../hdfs/server/namenode/FSNamesystem.java | 141 +-
.../hdfs/server/namenode/LeaseManager.java | 5 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 25 +-
.../hdfs/server/namenode/NameNodeMXBean.java | 7 -
.../hdfs/server/namenode/NameNodeRpcServer.java | 10 +-
.../hdfs/server/namenode/NamenodeFsck.java | 5 +
.../namenode/RetryStartFileException.java | 36 -
.../hdfs/server/namenode/TransferFsImage.java | 4 +-
.../ha/AbstractNNFailoverProxyProvider.java | 51 -
.../ha/ConfiguredFailoverProxyProvider.java | 2 +-
.../ha/WrappedFailoverProxyProvider.java | 80 -
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../hadoop/hdfs/tools/NNHAServiceTarget.java | 16 +-
.../hadoop/hdfs/util/StripedBlockUtil.java | 946 -----
.../main/native/fuse-dfs/test/TestFuseDFS.java | 4 +-
.../src/main/proto/DatanodeProtocol.proto | 1 +
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 198 ++
.../src/main/proto/InterDatanodeProtocol.proto | 1 +
.../src/main/proto/JournalProtocol.proto | 1 +
.../src/main/proto/NamenodeProtocol.proto | 1 +
.../src/main/proto/QJournalProtocol.proto | 1 +
.../src/main/resources/hdfs-default.xml | 58 +-
.../hadoop-hdfs/src/site/markdown/Federation.md | 2 +-
.../org/apache/hadoop/hdfs/TestFiPipelines.java | 17 +-
.../datanode/TestFiDataTransferProtocol.java | 4 +-
.../datanode/TestFiDataTransferProtocol2.java | 8 +-
.../apache/hadoop/fs/TestFcHdfsSetUMask.java | 4 +-
.../TestSWebHdfsFileContextMainOperations.java | 5 +
.../TestWebHdfsFileContextMainOperations.java | 5 +
.../apache/hadoop/hdfs/BenchmarkThroughput.java | 3 +-
.../apache/hadoop/hdfs/BlockReaderTestUtil.java | 2 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +-
.../hdfs/MiniDFSClusterWithNodeGroup.java | 2 +-
.../hadoop/hdfs/TestBlockReaderLocal.java | 2 +
.../hadoop/hdfs/TestBlockStoragePolicy.java | 1 -
.../TestClientProtocolForPipelineRecovery.java | 6 +-
.../apache/hadoop/hdfs/TestCrcCorruption.java | 2 +-
.../hadoop/hdfs/TestDFSClientRetries.java | 3 +-
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 5 +-
.../org/apache/hadoop/hdfs/TestDFSPacket.java | 25 +-
.../org/apache/hadoop/hdfs/TestDFSShell.java | 13 +
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 8 +-
.../apache/hadoop/hdfs/TestDecommission.java | 43 +
.../hadoop/hdfs/TestDistributedFileSystem.java | 3 +-
.../hadoop/hdfs/TestExternalBlockReader.java | 43 +-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 4 +-
.../hadoop/hdfs/TestFileConcurrentReader.java | 9 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 12 +-
.../hadoop/hdfs/TestFileCreationClient.java | 11 +-
.../org/apache/hadoop/hdfs/TestFileStatus.java | 7 +-
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 6 +-
.../org/apache/hadoop/hdfs/TestLargeBlock.java | 10 +-
.../java/org/apache/hadoop/hdfs/TestLease.java | 8 +-
.../apache/hadoop/hdfs/TestLeaseRecovery2.java | 13 +-
.../apache/hadoop/hdfs/TestListFilesInDFS.java | 4 +-
.../hadoop/hdfs/TestListFilesInFileContext.java | 4 +-
.../org/apache/hadoop/hdfs/TestLocalDFS.java | 7 +-
.../apache/hadoop/hdfs/TestPersistBlocks.java | 5 +-
.../java/org/apache/hadoop/hdfs/TestPread.java | 10 +-
.../hadoop/hdfs/TestReadWhileWriting.java | 7 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 24 +-
.../hdfs/qjournal/client/TestQJMWithFaults.java | 3 +-
.../client/TestQuorumJournalManager.java | 3 +-
.../client/TestQuorumJournalManagerUnit.java | 3 +-
.../TestClientProtocolWithDelegationToken.java | 12 +-
.../hdfs/security/TestDelegationToken.java | 4 +-
.../security/token/block/TestBlockToken.java | 12 +-
.../hdfs/server/balancer/TestBalancer.java | 3 +-
.../TestBalancerWithMultipleNameNodes.java | 4 +-
.../blockmanagement/TestBlockTokenWithDFS.java | 4 +-
.../TestBlocksWithNotEnoughRacks.java | 6 +-
.../blockmanagement/TestHeartbeatHandling.java | 27 +
.../TestPendingInvalidateBlock.java | 9 +-
.../blockmanagement/TestReplicationPolicy.java | 4 +-
.../server/datanode/TestBPOfferService.java | 3 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 5 +-
.../TestDataNodeVolumeFailureReporting.java | 5 +-
.../TestDatanodeProtocolRetryPolicy.java | 3 +-
.../server/datanode/TestDirectoryScanner.java | 234 +-
.../hdfs/server/datanode/TestTransferRbw.java | 4 +-
.../fsdataset/impl/TestSpaceReservation.java | 5 +-
.../fsdataset/impl/TestWriteToReplica.java | 4 +-
.../hdfs/server/mover/TestStorageMover.java | 13 +-
.../server/namenode/TestAuditLogAtDebug.java | 4 +-
.../hdfs/server/namenode/TestBackupNode.java | 5 +-
.../server/namenode/TestCacheDirectives.java | 4 +-
.../hdfs/server/namenode/TestCheckpoint.java | 3 +-
.../hdfs/server/namenode/TestEditLog.java | 3 +-
.../hdfs/server/namenode/TestEditLogRace.java | 4 +-
.../server/namenode/TestFSEditLogLoader.java | 6 +-
.../namenode/TestFSImageWithSnapshot.java | 4 +-
.../namenode/TestFavoredNodesEndToEnd.java | 5 +-
.../hdfs/server/namenode/TestFileTruncate.java | 5 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 8 +-
.../server/namenode/TestNameNodeMXBean.java | 3 -
.../namenode/TestNameNodeRetryCacheMetrics.java | 4 +-
.../hdfs/server/namenode/TestSaveNamespace.java | 3 +-
.../ha/TestDNFencingWithReplication.java | 11 +-
.../server/namenode/ha/TestEditLogTailer.java | 5 +-
.../hdfs/server/namenode/ha/TestHAFsck.java | 5 +-
.../namenode/ha/TestHAStateTransitions.java | 3 +-
.../ha/TestLossyRetryInvocationHandler.java | 8 +-
.../namenode/ha/TestRetryCacheWithHA.java | 4 +-
.../namenode/metrics/TestNameNodeMetrics.java | 6 +-
...tINodeFileUnderConstructionWithSnapshot.java | 12 +-
.../server/namenode/snapshot/TestSnapshot.java | 3 +-
.../hdfs/tools/TestDFSHAAdminMiniCluster.java | 4 +-
.../hadoop/hdfs/util/TestByteArrayManager.java | 11 +-
.../hdfs/web/TestFSMainOperationsWebHdfs.java | 4 +-
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 4 +-
.../web/TestWebHdfsWithMultipleNameNodes.java | 3 +-
.../hadoop/tools/TestHdfsConfigFields.java | 6 +-
.../apache/hadoop/tracing/TestTraceAdmin.java | 9 +-
.../org/apache/hadoop/tracing/TestTracing.java | 86 +-
.../TestTracingShortCircuitLocalRead.java | 18 +-
hadoop-mapreduce-project/CHANGES.txt | 17 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 4 +-
.../v2/app/job/impl/TestTaskAttempt.java | 8 +
.../v2/app/launcher/TestContainerLauncher.java | 11 +
.../app/launcher/TestContainerLauncherImpl.java | 9 +
.../app/local/TestLocalContainerAllocator.java | 6 +-
.../v2/app/rm/TestRMContainerAllocator.java | 19 +-
.../java/org/apache/hadoop/mapred/Master.java | 27 +-
.../org/apache/hadoop/mapred/TestMaster.java | 13 +
hadoop-project/pom.xml | 6 +-
.../dev-support/findbugs-exclude.xml | 32 +
hadoop-tools/hadoop-archive-logs/pom.xml | 18 +
.../apache/hadoop/tools/HadoopArchiveLogs.java | 243 +-
.../hadoop/tools/TestHadoopArchiveLogs.java | 231 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 +
.../hadoop/fs/azure/NativeAzureFileSystem.java | 3 -
.../fs/azure/NativeAzureFileSystemBaseTest.java | 1 -
.../src/site/markdown/DistCp.md.vm | 5 +-
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 14 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 13 +
.../sls/scheduler/ResourceSchedulerWrapper.java | 21 +-
.../sls/scheduler/SLSCapacityScheduler.java | 19 +-
.../org/apache/hadoop/streaming/UtilTest.java | 5 -
hadoop-yarn-project/CHANGES.txt | 80 +-
.../yarn/api/ContainerManagementProtocol.java | 30 +-
.../api/protocolrecords/AllocateRequest.java | 57 +-
.../api/protocolrecords/AllocateResponse.java | 42 +-
.../IncreaseContainersResourceRequest.java | 75 +
.../IncreaseContainersResourceResponse.java | 93 +
.../records/ContainerResourceChangeRequest.java | 117 +
.../api/records/ContainerResourceDecrease.java | 78 -
.../api/records/ContainerResourceIncrease.java | 84 -
.../ContainerResourceIncreaseRequest.java | 80 -
.../yarn/api/records/ContainerStatus.java | 13 +
.../proto/containermanagement_protocol.proto | 1 +
.../src/main/proto/yarn_protos.proto | 14 +-
.../src/main/proto/yarn_service_protos.proto | 16 +-
.../yarn/client/api/impl/YarnClientImpl.java | 7 +-
.../yarn/client/TestResourceTrackerOnHA.java | 2 +-
.../api/impl/TestAMRMClientOnRMRestart.java | 8 +-
...ContainerManagementProtocolPBClientImpl.java | 20 +
...ontainerManagementProtocolPBServiceImpl.java | 22 +
.../impl/pb/AllocateRequestPBImpl.java | 119 +-
.../impl/pb/AllocateResponsePBImpl.java | 175 +-
...IncreaseContainersResourceRequestPBImpl.java | 170 +
...ncreaseContainersResourceResponsePBImpl.java | 241 ++
.../ContainerResourceChangeRequestPBImpl.java | 141 +
.../pb/ContainerResourceDecreasePBImpl.java | 136 -
.../pb/ContainerResourceIncreasePBImpl.java | 171 -
.../ContainerResourceIncreaseRequestPBImpl.java | 141 -
.../records/impl/pb/ContainerStatusPBImpl.java | 31 +-
.../resource/DefaultResourceCalculator.java | 5 +
.../resource/DominantResourceCalculator.java | 6 +
.../yarn/util/resource/ResourceCalculator.java | 5 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../hadoop/yarn/TestContainerLaunchRPC.java | 8 +
.../yarn/TestContainerResourceIncreaseRPC.java | 162 +
.../java/org/apache/hadoop/yarn/TestRPC.java | 8 +
.../hadoop/yarn/api/TestAllocateRequest.java | 73 -
.../hadoop/yarn/api/TestAllocateResponse.java | 114 -
.../yarn/api/TestContainerResourceDecrease.java | 66 -
.../yarn/api/TestContainerResourceIncrease.java | 74 -
.../TestContainerResourceIncreaseRequest.java | 68 -
.../hadoop/yarn/api/TestPBImplRecords.java | 54 +-
.../util/resource/TestResourceCalculator.java | 30 +-
...pplicationHistoryManagerOnTimelineStore.java | 12 +
.../ApplicationHistoryServer.java | 13 +-
...pplicationHistoryManagerOnTimelineStore.java | 43 +-
.../TestApplicationHistoryServer.java | 4 +-
.../protocolrecords/NodeHeartbeatResponse.java | 5 +
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 77 +-
.../yarn/server/api/records/NodeStatus.java | 15 +-
.../api/records/impl/pb/NodeStatusPBImpl.java | 75 +-
.../metrics/ApplicationMetricsConstants.java | 3 +
.../hadoop/yarn/server/utils/BuilderUtils.java | 4 +-
.../main/proto/yarn_server_common_protos.proto | 3 +-
.../yarn_server_common_service_protos.proto | 1 +
.../hadoop/yarn/TestYarnServerApiClasses.java | 39 +-
.../CMgrDecreaseContainersResourceEvent.java | 37 +
.../nodemanager/ContainerManagerEventType.java | 1 +
.../hadoop/yarn/server/nodemanager/Context.java | 3 +
.../nodemanager/LocalDirsHandlerService.java | 33 +-
.../yarn/server/nodemanager/NodeManager.java | 10 +
.../nodemanager/NodeStatusUpdaterImpl.java | 57 +-
.../containermanager/ContainerManagerImpl.java | 302 +-
.../containermanager/container/Container.java | 2 +
.../container/ContainerImpl.java | 24 +-
.../logaggregation/LogAggregationService.java | 11 +-
.../ChangeMonitoringContainerResourceEvent.java | 37 +
.../monitor/ContainersMonitorEventType.java | 3 +-
.../monitor/ContainersMonitorImpl.java | 207 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 11 +
.../recovery/NMLeveldbStateStoreService.java | 22 +
.../recovery/NMNullStateStoreService.java | 6 +
.../recovery/NMStateStoreService.java | 15 +
.../nodemanager/DummyContainerManager.java | 6 +-
.../TestContainerManagerWithLCE.java | 33 +
.../TestLocalDirsHandlerService.java | 18 +
.../nodemanager/TestNodeManagerResync.java | 258 ++
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../amrmproxy/BaseAMRMProxyTest.java | 5 +
.../amrmproxy/MockResourceManagerFacade.java | 6 +-
.../BaseContainerManagerTest.java | 43 +-
.../containermanager/TestContainerManager.java | 288 +-
.../TestContainerManagerRecovery.java | 233 +-
.../TestLogAggregationService.java | 19 +
.../monitor/MockResourceCalculatorPlugin.java | 69 +
.../MockResourceCalculatorProcessTree.java | 57 +
.../TestContainersMonitorResourceChange.java | 248 ++
.../metrics/TestNodeManagerMetrics.java | 18 +-
.../recovery/NMMemoryStateStoreService.java | 11 +-
.../TestNMLeveldbStateStoreService.java | 11 +
.../nodemanager/webapp/MockContainer.java | 4 +
.../ApplicationMasterService.java | 22 +-
.../server/resourcemanager/ClientRMService.java | 30 +-
.../server/resourcemanager/RMAuditLogger.java | 2 +
.../server/resourcemanager/RMServerUtils.java | 164 +
.../resourcemanager/ResourceTrackerService.java | 7 +-
.../resourcemanager/amlauncher/AMLauncher.java | 27 +-
.../metrics/ApplicationUpdatedEvent.java | 54 +
.../metrics/SystemMetricsEventType.java | 1 +
.../metrics/SystemMetricsPublisher.java | 29 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 5 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 4 +-
.../rmcontainer/RMContainer.java | 4 +
.../RMContainerChangeResourceEvent.java | 44 +
.../rmcontainer/RMContainerEventType.java | 13 +-
.../rmcontainer/RMContainerImpl.java | 121 +-
.../RMContainerUpdatesAcquiredEvent.java | 35 +
.../server/resourcemanager/rmnode/RMNode.java | 9 +
.../rmnode/RMNodeDecreaseContainerEvent.java | 39 +
.../resourcemanager/rmnode/RMNodeEventType.java | 1 +
.../resourcemanager/rmnode/RMNodeImpl.java | 93 +
.../rmnode/RMNodeStatusEvent.java | 33 +-
.../scheduler/AbstractYarnScheduler.java | 150 +-
.../resourcemanager/scheduler/Allocation.java | 22 +-
.../scheduler/AppSchedulingInfo.java | 249 +-
.../resourcemanager/scheduler/QueueMetrics.java | 16 +-
.../scheduler/SchedContainerChangeRequest.java | 118 +
.../scheduler/SchedulerApplication.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 250 +-
.../scheduler/SchedulerNode.java | 31 +
.../scheduler/SchedulerUtils.java | 11 +-
.../scheduler/YarnScheduler.java | 14 +-
.../scheduler/capacity/AbstractCSQueue.java | 23 +-
.../scheduler/capacity/CSAssignment.java | 9 +
.../scheduler/capacity/CSQueue.java | 16 +
.../scheduler/capacity/CapacityScheduler.java | 87 +-
.../scheduler/capacity/LeafQueue.java | 127 +-
.../scheduler/capacity/ParentQueue.java | 115 +-
.../allocator/AbstractContainerAllocator.java | 131 +
.../capacity/allocator/ContainerAllocator.java | 149 +-
.../allocator/IncreaseContainerAllocator.java | 365 ++
.../allocator/RegularContainerAllocator.java | 30 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 68 +-
.../scheduler/fair/FSLeafQueue.java | 5 +-
.../scheduler/fair/FSParentQueue.java | 3 +-
.../scheduler/fair/FairScheduler.java | 35 +-
.../scheduler/fair/QueueManager.java | 24 +-
.../scheduler/fifo/FifoScheduler.java | 25 +-
.../server/resourcemanager/Application.java | 2 +-
.../yarn/server/resourcemanager/MockAM.java | 9 +
.../yarn/server/resourcemanager/MockNM.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 13 +
.../yarn/server/resourcemanager/MockRM.java | 25 +-
.../server/resourcemanager/NodeManager.java | 13 +-
.../resourcemanager/TestAMAuthorization.java | 8 +
.../resourcemanager/TestApplicationCleanup.java | 6 +-
.../TestApplicationMasterLauncher.java | 75 +
.../TestApplicationMasterService.java | 144 +-
.../resourcemanager/TestClientRMService.java | 36 +-
.../applicationsmanager/TestAMRestart.java | 15 +-
.../TestRMAppLogAggregationStatus.java | 10 +-
.../metrics/TestSystemMetricsPublisher.java | 50 +-
.../attempt/TestRMAppAttemptTransitions.java | 53 +-
.../rmcontainer/TestRMContainerImpl.java | 117 +-
.../capacity/TestCapacityScheduler.java | 130 +-
.../scheduler/capacity/TestChildQueueOrder.java | 4 +-
.../capacity/TestContainerAllocation.java | 50 +-
.../capacity/TestContainerResizing.java | 963 ++++++
.../scheduler/capacity/TestLeafQueue.java | 4 +-
.../scheduler/capacity/TestParentQueue.java | 4 +-
.../scheduler/capacity/TestReservations.java | 9 +-
.../scheduler/fair/FairSchedulerTestBase.java | 6 +-
.../fair/TestContinuousScheduling.java | 2 +-
.../scheduler/fair/TestFSParentQueue.java | 79 +
.../scheduler/fair/TestFairScheduler.java | 30 +-
.../scheduler/fifo/TestFifoScheduler.java | 32 +-
.../security/TestAMRMTokens.java | 3 +-
.../yarn/server/webproxy/AppReportFetcher.java | 49 +-
.../server/webproxy/WebAppProxyServlet.java | 46 +-
.../server/webproxy/TestWebAppProxyServlet.java | 113 +-
485 files changed, 31104 insertions(+), 23102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd55202/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd55202/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
----------------------------------------------------------------------
diff --cc
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
index a8a7494,648043e..616ff3a
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java
@@@ -196,12 -198,8 +198,12 @@@ abstract public class FSOutputSummer ex
return sum.getChecksumSize();
}
+ protected DataChecksum getDataChecksum() {
+ return sum;
+ }
+
protected TraceScope createWriteTraceScope() {
- return NullScope.INSTANCE;
+ return null;
}
/** Generate checksums for the given data chunks and output chunks &
checksums
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd55202/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd55202/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd55202/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocalLegacy.java
----------------------------------------------------------------------
