Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28d313d1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28d313d1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28d313d1 Branch: refs/heads/HDFS-7240 Commit: 28d313d19ab186c9245d0d350aacb24bb881ba89 Parents: 8a070ee 80d33b5 Author: Anu Engineer <aengin...@apache.org> Authored: Tue Sep 29 13:27:41 2015 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Tue Sep 29 13:27:41 2015 -0700 ---------------------------------------------------------------------- dev-support/test-patch.sh | 29 +- .../main/resources/assemblies/hadoop-tools.xml | 7 + .../server/AuthenticationToken.java | 3 +- hadoop-common-project/hadoop-common/CHANGES.txt | 250 +- hadoop-common-project/hadoop-common/pom.xml | 64 +- .../src/main/conf/log4j.properties | 17 +- .../org/apache/hadoop/conf/Configuration.java | 20 +- .../hadoop/fs/CommonConfigurationKeys.java | 3 + .../fs/CommonConfigurationKeysPublic.java | 19 + .../org/apache/hadoop/fs/FSOutputSummer.java | 12 +- .../java/org/apache/hadoop/fs/FileContext.java | 7 + .../java/org/apache/hadoop/fs/FileSystem.java | 27 +- .../java/org/apache/hadoop/fs/FileUtil.java | 14 +- .../org/apache/hadoop/fs/FilterFileSystem.java | 1 - .../main/java/org/apache/hadoop/fs/FsShell.java | 26 +- .../java/org/apache/hadoop/fs/FsTracer.java | 64 + .../main/java/org/apache/hadoop/fs/Globber.java | 15 +- .../org/apache/hadoop/fs/HarFileSystem.java | 1 - .../org/apache/hadoop/fs/LocalDirAllocator.java | 4 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 1 - .../main/java/org/apache/hadoop/fs/Trash.java | 3 +- .../hadoop/fs/shell/CommandWithDestination.java | 41 +- .../apache/hadoop/fs/shell/CopyCommands.java | 19 +- .../java/org/apache/hadoop/fs/shell/Delete.java | 68 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 1 - .../apache/hadoop/ha/ActiveStandbyElector.java | 4 + .../apache/hadoop/io/retry/RetryPolicies.java | 13 +- .../main/java/org/apache/hadoop/ipc/Client.java | 22 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 17 +- .../main/java/org/apache/hadoop/ipc/Server.java | 51 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 13 +- .../hadoop/metrics2/impl/MetricsConfig.java | 3 + .../metrics2/impl/MetricsSourceAdapter.java | 12 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 20 +- .../apache/hadoop/metrics2/package-info.java | 4 +- .../apache/hadoop/metrics2/sink/StatsDSink.java | 218 ++ .../main/java/org/apache/hadoop/net/DNS.java | 129 +- .../apache/hadoop/security/SaslPlainServer.java | 2 +- .../apache/hadoop/security/SecurityUtil.java | 37 +- .../security/authorize/AccessControlList.java | 2 +- .../apache/hadoop/tracing/SpanReceiverHost.java | 208 -- .../org/apache/hadoop/tracing/TraceUtils.java | 24 +- .../tracing/TracerConfigurationManager.java | 100 + .../main/java/org/apache/hadoop/util/GSet.java | 14 + .../org/apache/hadoop/util/GSetByHashMap.java | 6 + .../org/apache/hadoop/util/LightWeightGSet.java | 82 +- .../hadoop/util/LightWeightResizableGSet.java | 129 + .../java/org/apache/hadoop/util/LineReader.java | 17 +- .../java/org/apache/hadoop/util/ProtoUtil.java | 13 +- .../apache/hadoop/util/ShutdownHookManager.java | 6 + .../org/apache/hadoop/util/VersionInfo.java | 2 +- .../src/main/proto/RpcHeader.proto | 5 +- .../src/main/resources/core-default.xml | 65 +- .../src/site/markdown/FileSystemShell.md | 15 +- .../hadoop-common/src/site/markdown/Metrics.md | 2 + .../hadoop-common/src/site/markdown/Tracing.md | 91 +- .../java/org/apache/hadoop/cli/TestCLI.java | 2 +- .../org/apache/hadoop/cli/util/CLICommand.java | 5 +- .../org/apache/hadoop/cli/util/CLITestCmd.java | 6 +- .../apache/hadoop/conf/TestConfiguration.java | 2 +- .../java/org/apache/hadoop/fs/TestFsShell.java | 44 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 46 + .../org/apache/hadoop/fs/TestHarFileSystem.java | 1 - .../apache/hadoop/fs/TestLocalDirAllocator.java | 26 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 1 + .../AbstractContractRootDirectoryTest.java | 14 + .../java/org/apache/hadoop/fs/test-untar.tar | Bin 20480 -> 0 bytes .../java/org/apache/hadoop/fs/test-untar.tgz | Bin 2024 -> 0 bytes .../fs/viewfs/ViewFileSystemBaseTest.java | 2 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 2 +- .../apache/hadoop/ha/ClientBaseWithFixes.java | 40 +- .../http/TestAuthenticationSessionCookie.java | 5 +- .../apache/hadoop/http/TestHttpCookieFlag.java | 7 +- .../apache/hadoop/http/TestSSLHttpServer.java | 4 +- .../apache/hadoop/io/retry/TestRetryProxy.java | 13 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 38 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 4 +- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 9 +- .../metrics2/impl/TestMetricsSystemImpl.java | 13 + .../hadoop/metrics2/impl/TestStatsDMetrics.java | 122 + .../org/apache/hadoop/net/ServerSocketUtil.java | 7 +- .../java/org/apache/hadoop/net/TestDNS.java | 110 +- .../hadoop/security/TestSecurityUtil.java | 2 +- .../authorize/TestAccessControlList.java | 9 + .../hadoop/security/ssl/KeyStoreTestUtil.java | 60 +- .../delegation/web/TestWebDelegationToken.java | 7 +- .../apache/hadoop/tracing/SetSpanReceiver.java | 13 +- .../apache/hadoop/tracing/TestTraceUtils.java | 2 +- .../java/org/apache/hadoop/util/TestGSet.java | 69 +- .../hadoop/util/TestLightWeightCache.java | 6 + .../util/TestLightWeightResizableGSet.java | 252 ++ .../src/test/resources/test-untar.tar | Bin 0 -> 20480 bytes .../src/test/resources/test-untar.tgz | Bin 0 -> 2024 bytes .../src/test/resources/testConf.xml | 22 +- .../dev-support/findbugsExcludeFile.xml | 35 + hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 5 + .../org/apache/hadoop/fs/HdfsBlockLocation.java | 47 + .../hadoop/hdfs/BlockMissingException.java | 65 + .../org/apache/hadoop/hdfs/BlockReader.java | 102 + .../apache/hadoop/hdfs/BlockReaderFactory.java | 906 +++++ .../apache/hadoop/hdfs/BlockReaderLocal.java | 754 +++++ .../hadoop/hdfs/BlockReaderLocalLegacy.java | 738 ++++ .../org/apache/hadoop/hdfs/BlockReaderUtil.java | 57 + .../org/apache/hadoop/hdfs/ClientContext.java | 196 ++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 3135 +++++++++++++++++ .../hadoop/hdfs/DFSClientFaultInjector.java | 60 + .../hadoop/hdfs/DFSHedgedReadMetrics.java | 58 + .../hadoop/hdfs/DFSInotifyEventInputStream.java | 234 ++ .../org/apache/hadoop/hdfs/DFSInputStream.java | 1917 +++++++++++ .../org/apache/hadoop/hdfs/DFSOutputStream.java | 917 +++++ .../java/org/apache/hadoop/hdfs/DFSPacket.java | 350 ++ .../org/apache/hadoop/hdfs/DFSUtilClient.java | 223 ++ .../org/apache/hadoop/hdfs/DataStreamer.java | 1947 +++++++++++ .../hadoop/hdfs/DistributedFileSystem.java | 2259 +++++++++++++ .../apache/hadoop/hdfs/ExternalBlockReader.java | 126 + .../hadoop/hdfs/HdfsConfigurationLoader.java | 44 + .../apache/hadoop/hdfs/KeyProviderCache.java | 112 + .../hadoop/hdfs/NameNodeProxiesClient.java | 366 ++ .../java/org/apache/hadoop/hdfs/PeerCache.java | 291 ++ .../apache/hadoop/hdfs/RemoteBlockReader.java | 514 +++ .../apache/hadoop/hdfs/RemoteBlockReader2.java | 483 +++ .../apache/hadoop/hdfs/RemotePeerFactory.java | 43 + .../org/apache/hadoop/hdfs/ReplicaAccessor.java | 10 +- .../hadoop/hdfs/ReplicaAccessorBuilder.java | 3 + .../hdfs/UnknownCipherSuiteException.java | 35 + .../UnknownCryptoProtocolVersionException.java | 38 + .../org/apache/hadoop/hdfs/XAttrHelper.java | 174 + .../hadoop/hdfs/client/BlockReportOptions.java | 59 + .../hdfs/client/HdfsClientConfigKeys.java | 38 +- .../hadoop/hdfs/client/HdfsDataInputStream.java | 113 + .../hdfs/client/HdfsDataOutputStream.java | 112 + .../client/impl/CorruptFileBlockIterator.java | 105 + .../hadoop/hdfs/client/impl/DfsClientConf.java | 64 +- .../hadoop/hdfs/client/impl/LeaseRenewer.java | 524 +++ .../hdfs/inotify/MissingEventsException.java | 54 + .../apache/hadoop/hdfs/net/BasicInetPeer.java | 133 + .../apache/hadoop/hdfs/net/EncryptedPeer.java | 142 + .../org/apache/hadoop/hdfs/net/NioInetPeer.java | 136 + .../java/org/apache/hadoop/hdfs/net/Peer.java | 8 +- .../hadoop/hdfs/protocol/AclException.java | 39 + .../hdfs/protocol/BlockLocalPathInfo.java | 70 + .../hdfs/protocol/CacheDirectiveIterator.java | 129 + .../hadoop/hdfs/protocol/CachePoolIterator.java | 62 + .../hdfs/protocol/ClientDatanodeProtocol.java | 159 + .../hadoop/hdfs/protocol/DatanodeInfo.java | 47 +- .../hdfs/protocol/EncryptionZoneIterator.java | 62 + .../hadoop/hdfs/protocol/HdfsConstants.java | 31 + .../hdfs/protocol/HdfsLocatedFileStatus.java | 83 + .../QuotaByStorageTypeExceededException.java | 56 + .../hdfs/protocol/UnresolvedPathException.java | 87 + .../datatransfer/DataTransferProtoUtil.java | 43 +- .../protocol/datatransfer/IOStreamPair.java | 37 + .../InvalidEncryptionKeyException.java | 40 + .../protocol/datatransfer/PacketHeader.java | 214 ++ .../protocol/datatransfer/PacketReceiver.java | 310 ++ .../hdfs/protocol/datatransfer/PipelineAck.java | 243 ++ .../datatransfer/ReplaceDatanodeOnFailure.java | 200 ++ .../hdfs/protocol/datatransfer/Sender.java | 22 +- .../datatransfer/TrustedChannelResolver.java | 81 + .../sasl/DataEncryptionKeyFactory.java | 38 + .../datatransfer/sasl/DataTransferSaslUtil.java | 519 +++ .../sasl/SaslDataTransferClient.java | 498 +++ .../datatransfer/sasl/SaslParticipant.java | 210 ++ .../SaslResponseWithNegotiatedCipherOption.java | 33 + .../protocolPB/ClientDatanodeProtocolPB.java | 37 + .../ClientDatanodeProtocolTranslatorPB.java | 343 ++ .../protocolPB/ClientNamenodeProtocolPB.java | 46 + .../ClientNamenodeProtocolTranslatorPB.java | 1531 +++++++++ .../hadoop/hdfs/protocolPB/PBHelperClient.java | 2093 +++++++++++- .../token/block/BlockTokenSelector.java | 48 + .../server/datanode/BlockMetadataHeader.java | 209 ++ .../datanode/ReplicaNotFoundException.java | 53 + .../namenode/RetryStartFileException.java | 36 + .../ha/AbstractNNFailoverProxyProvider.java | 55 + .../ha/WrappedFailoverProxyProvider.java | 71 + .../hadoop/hdfs/shortcircuit/ClientMmap.java | 75 + .../hdfs/shortcircuit/DomainSocketFactory.java | 196 ++ .../hdfs/shortcircuit/ShortCircuitCache.java | 1066 ++++++ .../hdfs/shortcircuit/ShortCircuitReplica.java | 352 ++ .../shortcircuit/ShortCircuitReplicaInfo.java | 64 + .../hdfs/util/ByteBufferOutputStream.java | 49 + .../apache/hadoop/hdfs/util/IOUtilsClient.java | 46 + .../apache/hadoop/hdfs/util/LongBitFormat.java | 71 + .../hadoop/hdfs/web/ByteRangeInputStream.java | 11 + .../apache/hadoop/hdfs/web/JsonUtilClient.java | 3 +- .../hadoop/hdfs/web/URLConnectionFactory.java | 30 +- .../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +- .../hdfs/web/oauth2/AccessTokenProvider.java | 66 + .../hdfs/web/oauth2/AccessTokenTimer.java | 103 + .../ConfCredentialBasedAccessTokenProvider.java | 62 + ...onfRefreshTokenBasedAccessTokenProvider.java | 146 + .../CredentialBasedAccessTokenProvider.java | 135 + .../oauth2/OAuth2ConnectionConfigurator.java | 79 + .../hadoop/hdfs/web/oauth2/OAuth2Constants.java | 46 + .../apache/hadoop/hdfs/web/oauth2/Utils.java | 63 + .../hadoop/hdfs/web/oauth2/package-info.java | 26 + .../src/main/proto/ClientDatanodeProtocol.proto | 16 + .../src/main/proto/hdfs.proto | 203 +- .../services/org.apache.hadoop.fs.FileSystem | 18 + .../hadoop/hdfs/nfs/mount/RpcProgramMountd.java | 4 +- .../hadoop/hdfs/nfs/nfs3/DFSClientCache.java | 4 +- .../hadoop/hdfs/nfs/nfs3/OffsetRange.java | 4 + .../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 141 +- .../apache/hadoop/hdfs/nfs/nfs3/WriteCtx.java | 82 +- .../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 99 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 544 ++- .../dev-support/findbugsExcludeFile.xml | 29 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 9 +- .../hadoop-hdfs/src/contrib/bkjournal/pom.xml | 1 + .../bkjournal/BookKeeperEditLogInputStream.java | 2 +- .../bkjournal/src/main/proto/bkjournal.proto | 1 + .../hdfs/server/namenode/FSEditLogTestUtil.java | 3 +- .../main/java/org/apache/hadoop/fs/Hdfs.java | 7 +- .../org/apache/hadoop/fs/HdfsBlockLocation.java | 47 - .../java/org/apache/hadoop/fs/SWebHdfs.java | 15 +- .../main/java/org/apache/hadoop/fs/WebHdfs.java | 14 +- .../hadoop/hdfs/BlockMissingException.java | 65 - .../org/apache/hadoop/hdfs/BlockReader.java | 102 - .../apache/hadoop/hdfs/BlockReaderFactory.java | 894 ----- .../apache/hadoop/hdfs/BlockReaderLocal.java | 741 ---- .../hadoop/hdfs/BlockReaderLocalLegacy.java | 735 ---- .../org/apache/hadoop/hdfs/BlockReaderUtil.java | 57 - .../org/apache/hadoop/hdfs/ClientContext.java | 195 -- .../java/org/apache/hadoop/hdfs/DFSClient.java | 3167 ------------------ .../hadoop/hdfs/DFSClientFaultInjector.java | 57 - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 104 +- .../hadoop/hdfs/DFSHedgedReadMetrics.java | 58 - .../hadoop/hdfs/DFSInotifyEventInputStream.java | 239 -- .../org/apache/hadoop/hdfs/DFSInputStream.java | 1915 ----------- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 911 ----- .../java/org/apache/hadoop/hdfs/DFSPacket.java | 345 -- .../java/org/apache/hadoop/hdfs/DFSUtil.java | 101 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 1909 ----------- .../hadoop/hdfs/DistributedFileSystem.java | 2262 ------------- .../apache/hadoop/hdfs/ExternalBlockReader.java | 120 - .../java/org/apache/hadoop/hdfs/HAUtil.java | 9 +- .../apache/hadoop/hdfs/HdfsConfiguration.java | 11 +- .../apache/hadoop/hdfs/KeyProviderCache.java | 111 - .../org/apache/hadoop/hdfs/NameNodeProxies.java | 278 +- .../java/org/apache/hadoop/hdfs/PeerCache.java | 290 -- .../apache/hadoop/hdfs/RemoteBlockReader.java | 508 --- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 477 --- .../apache/hadoop/hdfs/RemotePeerFactory.java | 43 - .../hdfs/UnknownCipherSuiteException.java | 35 - .../UnknownCryptoProtocolVersionException.java | 38 - .../org/apache/hadoop/hdfs/XAttrHelper.java | 174 - .../hadoop/hdfs/client/BlockReportOptions.java | 59 - .../hadoop/hdfs/client/HdfsDataInputStream.java | 113 - .../hdfs/client/HdfsDataOutputStream.java | 112 - .../client/impl/CorruptFileBlockIterator.java | 105 - .../hadoop/hdfs/client/impl/LeaseRenewer.java | 524 --- .../hdfs/inotify/MissingEventsException.java | 54 - .../apache/hadoop/hdfs/net/BasicInetPeer.java | 133 - .../hadoop/hdfs/net/DomainPeerServer.java | 5 + .../apache/hadoop/hdfs/net/EncryptedPeer.java | 142 - .../org/apache/hadoop/hdfs/net/NioInetPeer.java | 136 - .../org/apache/hadoop/hdfs/net/PeerServer.java | 9 +- .../apache/hadoop/hdfs/net/TcpPeerServer.java | 70 +- .../hadoop/hdfs/protocol/AclException.java | 39 - .../hdfs/protocol/BlockLocalPathInfo.java | 70 - .../hdfs/protocol/CacheDirectiveIterator.java | 130 - .../hadoop/hdfs/protocol/CachePoolIterator.java | 63 - .../hdfs/protocol/ClientDatanodeProtocol.java | 152 - .../hdfs/protocol/EncryptionZoneIterator.java | 64 - .../hdfs/protocol/HdfsLocatedFileStatus.java | 83 - .../hadoop/hdfs/protocol/LayoutVersion.java | 2 +- .../QuotaByStorageTypeExceededException.java | 56 - .../hdfs/protocol/UnresolvedPathException.java | 87 - .../protocol/datatransfer/IOStreamPair.java | 37 - .../InvalidEncryptionKeyException.java | 40 - .../protocol/datatransfer/PacketHeader.java | 214 -- .../protocol/datatransfer/PacketReceiver.java | 310 -- .../hdfs/protocol/datatransfer/PipelineAck.java | 274 -- .../hdfs/protocol/datatransfer/Receiver.java | 83 +- .../datatransfer/ReplaceDatanodeOnFailure.java | 200 -- .../datatransfer/TrustedChannelResolver.java | 81 - .../sasl/DataEncryptionKeyFactory.java | 38 - .../datatransfer/sasl/DataTransferSaslUtil.java | 519 --- .../sasl/SaslDataTransferClient.java | 498 --- .../sasl/SaslDataTransferServer.java | 2 +- .../datatransfer/sasl/SaslParticipant.java | 210 -- .../SaslResponseWithNegotiatedCipherOption.java | 33 - .../protocolPB/ClientDatanodeProtocolPB.java | 37 - ...tDatanodeProtocolServerSideTranslatorPB.java | 26 +- .../ClientDatanodeProtocolTranslatorPB.java | 326 -- .../protocolPB/ClientNamenodeProtocolPB.java | 46 - ...tNamenodeProtocolServerSideTranslatorPB.java | 142 +- .../ClientNamenodeProtocolTranslatorPB.java | 1531 --------- .../DatanodeProtocolClientSideTranslatorPB.java | 12 +- .../DatanodeProtocolServerSideTranslatorPB.java | 18 +- ...rDatanodeProtocolServerSideTranslatorPB.java | 4 +- .../protocolPB/JournalProtocolTranslatorPB.java | 2 +- .../NamenodeProtocolServerSideTranslatorPB.java | 7 +- .../NamenodeProtocolTranslatorPB.java | 5 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 2194 +----------- .../QJournalProtocolTranslatorPB.java | 3 +- .../hadoop/hdfs/qjournal/server/Journal.java | 40 +- .../hdfs/qjournal/server/JournalMetrics.java | 7 +- .../hdfs/qjournal/server/JournalNode.java | 13 +- .../qjournal/server/JournalNodeRpcServer.java | 1 + .../token/block/BlockTokenSelector.java | 48 - .../hadoop/hdfs/server/balancer/Balancer.java | 170 +- .../server/balancer/BalancerParameters.java | 168 + .../hadoop/hdfs/server/balancer/Dispatcher.java | 3 +- .../server/blockmanagement/BlockCollection.java | 5 + .../hdfs/server/blockmanagement/BlockInfo.java | 44 +- .../blockmanagement/BlockInfoContiguous.java | 15 - .../server/blockmanagement/BlockManager.java | 534 ++- .../BlockPlacementPolicyDefault.java | 147 +- .../BlockPlacementPolicyWithNodeGroup.java | 14 +- .../BlockStoragePolicySuite.java | 13 +- .../blockmanagement/BlockToMarkCorrupt.java | 87 + .../hdfs/server/blockmanagement/BlocksMap.java | 26 +- .../blockmanagement/DatanodeDescriptor.java | 35 +- .../server/blockmanagement/DatanodeManager.java | 58 +- .../blockmanagement/DecommissionManager.java | 9 +- .../blockmanagement/HeartbeatManager.java | 30 +- .../server/blockmanagement/HostFileManager.java | 19 + .../server/blockmanagement/ReplicationWork.java | 87 + .../SequentialBlockIdGenerator.java | 5 +- .../hdfs/server/common/HdfsServerConstants.java | 33 +- .../hdfs/server/common/MetricsLoggerTask.java | 174 + .../hdfs/server/datanode/BPOfferService.java | 3 +- .../hdfs/server/datanode/BPServiceActor.java | 7 +- .../server/datanode/BlockMetadataHeader.java | 211 -- .../hdfs/server/datanode/BlockReceiver.java | 11 +- .../hdfs/server/datanode/BlockSender.java | 13 +- .../hadoop/hdfs/server/datanode/DNConf.java | 26 +- .../hadoop/hdfs/server/datanode/DataNode.java | 203 +- .../hdfs/server/datanode/DataNodeMXBean.java | 9 +- .../hdfs/server/datanode/DataXceiver.java | 27 +- .../hdfs/server/datanode/DataXceiverServer.java | 7 +- .../hdfs/server/datanode/DirectoryScanner.java | 338 +- .../hdfs/server/datanode/FinalizedReplica.java | 15 +- .../hdfs/server/datanode/ReplicaInfo.java | 109 +- .../datanode/ReplicaNotFoundException.java | 53 - .../server/datanode/ReplicaUnderRecovery.java | 10 - .../datanode/ReplicaWaitingToBeRecovered.java | 15 +- .../server/datanode/fsdataset/FsVolumeSpi.java | 8 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 11 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 41 +- .../datanode/fsdataset/impl/FsVolumeImpl.java | 72 +- .../impl/RamDiskAsyncLazyPersistService.java | 16 +- .../datanode/fsdataset/impl/ReplicaMap.java | 38 +- .../hdfs/server/namenode/BackupImage.java | 8 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 3 +- .../hdfs/server/namenode/CacheManager.java | 4 +- .../hadoop/hdfs/server/namenode/DfsServlet.java | 3 +- .../namenode/EditLogBackupInputStream.java | 2 +- .../server/namenode/EditLogFileInputStream.java | 62 +- .../server/namenode/EncryptionZoneManager.java | 4 +- .../hdfs/server/namenode/FSDirConcatOp.java | 2 +- .../hdfs/server/namenode/FSDirDeleteOp.java | 40 + .../server/namenode/FSDirEncryptionZoneOp.java | 303 ++ .../server/namenode/FSDirStatAndListingOp.java | 30 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 6 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 9 +- .../hdfs/server/namenode/FSDirectory.java | 344 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 12 +- .../hdfs/server/namenode/FSEditLogLoader.java | 65 +- .../hdfs/server/namenode/FSEditLogOp.java | 374 ++- .../hadoop/hdfs/server/namenode/FSImage.java | 80 +- .../server/namenode/FSImageFormatPBINode.java | 7 +- .../hdfs/server/namenode/FSNamesystem.java | 254 +- .../server/namenode/FSPermissionChecker.java | 41 +- .../server/namenode/FileJournalManager.java | 47 +- .../hadoop/hdfs/server/namenode/INode.java | 14 +- .../hdfs/server/namenode/INodeDirectory.java | 14 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 27 +- .../hadoop/hdfs/server/namenode/INodeId.java | 1 + .../hadoop/hdfs/server/namenode/INodeMap.java | 2 +- .../hdfs/server/namenode/INodeReference.java | 10 +- .../hdfs/server/namenode/INodeSymlink.java | 2 +- .../hdfs/server/namenode/ImageServlet.java | 5 +- .../hdfs/server/namenode/LeaseManager.java | 11 +- .../hadoop/hdfs/server/namenode/NameNode.java | 217 +- .../hdfs/server/namenode/NameNodeMXBean.java | 7 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 10 +- .../hdfs/server/namenode/NamenodeFsck.java | 18 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 5 + .../hdfs/server/namenode/QuotaCounts.java | 10 +- .../namenode/RetryStartFileException.java | 36 - .../hadoop/hdfs/server/namenode/SafeMode.java | 3 - .../hdfs/server/namenode/SecondaryNameNode.java | 2 +- .../hdfs/server/namenode/TransferFsImage.java | 4 +- .../hdfs/server/namenode/XAttrFormat.java | 14 +- .../ha/AbstractNNFailoverProxyProvider.java | 51 - .../server/namenode/ha/BootstrapStandby.java | 5 +- .../ha/ConfiguredFailoverProxyProvider.java | 2 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 2 +- .../namenode/ha/IPFailoverProxyProvider.java | 4 +- .../ha/WrappedFailoverProxyProvider.java | 80 - .../snapshot/DirectorySnapshottableFeature.java | 16 +- .../snapshot/DirectoryWithSnapshotFeature.java | 5 +- .../snapshot/FSImageFormatPBSnapshot.java | 6 +- .../hdfs/server/namenode/snapshot/Snapshot.java | 3 +- .../hadoop/hdfs/shortcircuit/ClientMmap.java | 75 - .../hdfs/shortcircuit/DomainSocketFactory.java | 194 -- .../hdfs/shortcircuit/ShortCircuitCache.java | 1068 ------ .../hdfs/shortcircuit/ShortCircuitReplica.java | 349 -- .../shortcircuit/ShortCircuitReplicaInfo.java | 64 - .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 52 +- .../hdfs/tools/DFSZKFailoverController.java | 3 +- .../org/apache/hadoop/hdfs/tools/GetGroups.java | 4 +- .../hadoop/hdfs/tools/NNHAServiceTarget.java | 19 +- .../hdfs/util/ByteBufferOutputStream.java | 49 - .../apache/hadoop/hdfs/util/LongBitFormat.java | 71 - .../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 + .../StorageContainerManager.java | 5 +- .../storagecontainer/StorageContainerMap.java | 6 + .../StorageContainerNameService.java | 16 +- .../main/native/fuse-dfs/test/TestFuseDFS.java | 4 +- .../src/main/proto/DatanodeProtocol.proto | 1 + .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 209 ++ .../src/main/proto/InterDatanodeProtocol.proto | 1 + .../src/main/proto/JournalProtocol.proto | 1 + .../src/main/proto/NamenodeProtocol.proto | 1 + .../src/main/proto/QJournalProtocol.proto | 1 + .../services/org.apache.hadoop.fs.FileSystem | 18 - .../src/main/resources/hdfs-default.xml | 110 +- .../src/main/webapps/hdfs/dfshealth.html | 7 +- .../src/main/webapps/hdfs/dfshealth.js | 9 +- .../src/main/webapps/hdfs/explorer.html | 76 +- .../src/main/webapps/hdfs/explorer.js | 89 +- .../src/main/webapps/static/dfs-dust.js | 12 +- .../src/main/webapps/static/hadoop.css | 22 + .../src/site/markdown/ArchivalStorage.md | 4 +- .../hadoop-hdfs/src/site/markdown/Federation.md | 2 +- .../src/site/markdown/HDFSCommands.md | 6 +- .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 6 +- .../src/site/markdown/HdfsRollingUpgrade.md | 293 ++ .../src/site/markdown/HdfsSnapshots.md | 301 ++ .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 25 + .../site/resources/images/LazyPersistWrites.png | Bin 107161 -> 90224 bytes .../src/site/xdoc/HdfsRollingUpgrade.xml | 329 -- .../hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml | 303 -- .../org/apache/hadoop/hdfs/TestFiPipelines.java | 17 +- .../datanode/TestFiDataTransferProtocol.java | 4 +- .../datanode/TestFiDataTransferProtocol2.java | 8 +- .../org/apache/hadoop/cli/CLITestCmdDFS.java | 8 +- .../java/org/apache/hadoop/cli/TestAclCLI.java | 2 +- .../apache/hadoop/cli/TestCacheAdminCLI.java | 7 +- .../apache/hadoop/cli/TestCryptoAdminCLI.java | 6 +- .../org/apache/hadoop/cli/TestDeleteCLI.java | 92 + .../java/org/apache/hadoop/cli/TestHDFSCLI.java | 4 +- .../org/apache/hadoop/cli/TestXAttrCLI.java | 2 +- .../apache/hadoop/fs/TestFcHdfsSetUMask.java | 4 +- .../TestSWebHdfsFileContextMainOperations.java | 5 + .../TestWebHdfsFileContextMainOperations.java | 5 + .../hadoop/fs/permission/TestStickyBit.java | 3 + .../apache/hadoop/hdfs/BenchmarkThroughput.java | 3 +- .../apache/hadoop/hdfs/BlockReaderTestUtil.java | 5 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 25 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 4 +- .../hdfs/MiniDFSClusterWithNodeGroup.java | 2 +- .../hadoop/hdfs/TestAppendSnapshotTruncate.java | 3 +- .../hadoop/hdfs/TestBalancerBandwidth.java | 55 +- .../hadoop/hdfs/TestBlockReaderLocal.java | 32 +- .../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 2 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 25 +- .../hdfs/TestClientBlockVerification.java | 4 +- .../TestClientProtocolForPipelineRecovery.java | 6 +- .../apache/hadoop/hdfs/TestCrcCorruption.java | 2 +- .../hadoop/hdfs/TestDFSClientFailover.java | 5 +- .../hadoop/hdfs/TestDFSClientRetries.java | 5 +- .../hdfs/TestDFSInotifyEventInputStream.java | 2 +- .../apache/hadoop/hdfs/TestDFSOutputStream.java | 5 +- .../org/apache/hadoop/hdfs/TestDFSPacket.java | 25 +- .../apache/hadoop/hdfs/TestDFSPermission.java | 42 +- .../org/apache/hadoop/hdfs/TestDFSShell.java | 69 + .../hadoop/hdfs/TestDFSShellGenericOptions.java | 13 +- .../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 78 +- .../hadoop/hdfs/TestDFSUpgradeFromImage.java | 107 +- .../org/apache/hadoop/hdfs/TestDFSUtil.java | 8 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 2 +- .../apache/hadoop/hdfs/TestDecommission.java | 58 +- .../hadoop/hdfs/TestDefaultNameNodePort.java | 30 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 57 +- .../hadoop/hdfs/TestEncryptedTransfer.java | 4 +- .../hadoop/hdfs/TestExternalBlockReader.java | 43 +- .../org/apache/hadoop/hdfs/TestFileAppend.java | 72 - .../org/apache/hadoop/hdfs/TestFileAppend4.java | 4 +- .../hadoop/hdfs/TestFileConcurrentReader.java | 9 +- .../apache/hadoop/hdfs/TestFileCreation.java | 12 +- .../hadoop/hdfs/TestFileCreationClient.java | 11 +- .../org/apache/hadoop/hdfs/TestFileStatus.java | 12 +- .../org/apache/hadoop/hdfs/TestGetBlocks.java | 7 +- .../java/org/apache/hadoop/hdfs/TestHFlush.java | 6 +- .../org/apache/hadoop/hdfs/TestLargeBlock.java | 10 +- .../java/org/apache/hadoop/hdfs/TestLease.java | 8 +- .../apache/hadoop/hdfs/TestLeaseRecovery2.java | 13 +- .../apache/hadoop/hdfs/TestListFilesInDFS.java | 4 +- .../hadoop/hdfs/TestListFilesInFileContext.java | 4 +- .../org/apache/hadoop/hdfs/TestLocalDFS.java | 7 +- .../apache/hadoop/hdfs/TestPersistBlocks.java | 10 +- .../java/org/apache/hadoop/hdfs/TestPread.java | 10 +- .../java/org/apache/hadoop/hdfs/TestQuota.java | 32 + .../hadoop/hdfs/TestReadWhileWriting.java | 7 +- .../sasl/SaslDataTransferTestCase.java | 2 +- .../datatransfer/sasl/TestSaslDataTransfer.java | 2 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 60 +- .../hdfs/qjournal/TestSecureNNWithQJM.java | 2 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 3 +- .../client/TestQuorumJournalManager.java | 3 +- .../client/TestQuorumJournalManagerUnit.java | 3 +- .../hdfs/qjournal/server/TestJournalNode.java | 9 + .../TestClientProtocolWithDelegationToken.java | 12 +- .../hdfs/security/TestDelegationToken.java | 4 +- .../security/token/block/TestBlockToken.java | 22 +- .../hdfs/server/balancer/TestBalancer.java | 200 +- .../balancer/TestBalancerWithHANameNodes.java | 8 +- .../TestBalancerWithMultipleNameNodes.java | 181 +- .../balancer/TestBalancerWithNodeGroup.java | 4 +- .../server/blockmanagement/TestBlockInfo.java | 11 +- .../blockmanagement/TestBlockManager.java | 15 + .../blockmanagement/TestBlockTokenWithDFS.java | 10 +- .../TestBlocksWithNotEnoughRacks.java | 6 +- .../blockmanagement/TestDatanodeManager.java | 103 +- .../blockmanagement/TestHeartbeatHandling.java | 27 + .../blockmanagement/TestHostFileManager.java | 7 +- .../TestPendingInvalidateBlock.java | 9 +- .../blockmanagement/TestReplicationPolicy.java | 43 +- .../TestReplicationPolicyWithNodeGroup.java | 19 + .../hdfs/server/datanode/DataNodeTestUtils.java | 86 +- .../server/datanode/SimulatedFSDataset.java | 2 +- .../server/datanode/TestBPOfferService.java | 3 +- .../hdfs/server/datanode/TestBlockRecovery.java | 5 +- .../server/datanode/TestCachingStrategy.java | 11 +- .../server/datanode/TestDataNodeMXBean.java | 6 + .../datanode/TestDataNodeMetricsLogger.java | 224 ++ .../datanode/TestDataNodeRollingUpgrade.java | 8 +- .../TestDataNodeTransferSocketSize.java | 71 + .../datanode/TestDataNodeVolumeFailure.java | 6 +- .../TestDataNodeVolumeFailureReporting.java | 5 +- .../TestDatanodeProtocolRetryPolicy.java | 3 +- .../server/datanode/TestDirectoryScanner.java | 236 +- .../hdfs/server/datanode/TestTransferRbw.java | 4 +- .../datanode/extdataset/ExternalVolumeImpl.java | 2 +- .../fsdataset/impl/FsDatasetTestUtil.java | 6 - .../fsdataset/impl/LazyPersistTestCase.java | 16 +- .../fsdataset/impl/TestDatanodeRestart.java | 72 - .../datanode/fsdataset/impl/TestLazyWriter.java | 1 + .../fsdataset/impl/TestRbwSpaceReservation.java | 452 --- .../fsdataset/impl/TestSpaceReservation.java | 575 ++++ .../fsdataset/impl/TestWriteToReplica.java | 4 +- .../hdfs/server/mover/TestStorageMover.java | 13 +- .../server/namenode/NNThroughputBenchmark.java | 3 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../server/namenode/TestAuditLogAtDebug.java | 4 +- .../hdfs/server/namenode/TestBackupNode.java | 5 +- .../server/namenode/TestCacheDirectives.java | 4 +- .../TestCheckPointForSecurityTokens.java | 4 +- .../hdfs/server/namenode/TestCheckpoint.java | 3 +- .../TestCommitBlockSynchronization.java | 9 +- .../namenode/TestDiskspaceQuotaUpdate.java | 61 + .../hdfs/server/namenode/TestEditLog.java | 108 +- .../namenode/TestEditLogFileInputStream.java | 80 + .../hdfs/server/namenode/TestEditLogRace.java | 4 +- .../server/namenode/TestFSEditLogLoader.java | 19 +- .../namenode/TestFSImageWithSnapshot.java | 9 +- .../hdfs/server/namenode/TestFSNamesystem.java | 25 +- .../namenode/TestFavoredNodesEndToEnd.java | 5 +- .../hdfs/server/namenode/TestFileTruncate.java | 10 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 62 +- .../hdfs/server/namenode/TestINodeFile.java | 4 +- .../namenode/TestListCorruptFileBlocks.java | 3 +- .../hdfs/server/namenode/TestMetaSave.java | 11 + .../server/namenode/TestNameNodeMXBean.java | 23 +- .../namenode/TestNameNodeMetricsLogger.java | 5 +- .../namenode/TestNameNodeRetryCacheMetrics.java | 4 +- .../namenode/TestProtectedDirectories.java | 373 +++ .../hdfs/server/namenode/TestSaveNamespace.java | 3 +- .../server/namenode/TestStorageRestore.java | 3 +- .../hdfs/server/namenode/TestXAttrFeature.java | 12 + .../ha/TestDNFencingWithReplication.java | 11 +- .../server/namenode/ha/TestEditLogTailer.java | 5 +- .../namenode/ha/TestFailureToReadEdits.java | 3 +- .../hdfs/server/namenode/ha/TestHAFsck.java | 5 +- .../namenode/ha/TestHAStateTransitions.java | 3 +- .../ha/TestLossyRetryInvocationHandler.java | 8 +- .../namenode/ha/TestRetryCacheWithHA.java | 4 +- .../namenode/metrics/TestNameNodeMetrics.java | 6 +- .../TestGetContentSummaryWithSnapshot.java | 126 + ...tINodeFileUnderConstructionWithSnapshot.java | 12 +- .../server/namenode/snapshot/TestSnapshot.java | 3 +- .../snapshot/TestSnapshotBlocksMap.java | 30 +- .../namenode/snapshot/TestSnapshotDeletion.java | 9 +- .../shortcircuit/TestShortCircuitCache.java | 5 +- .../shortcircuit/TestShortCircuitLocalRead.java | 104 +- .../hadoop/hdfs/tools/TestDFSAdminWithHA.java | 7 + .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 4 +- .../hadoop/hdfs/util/TestByteArrayManager.java | 11 +- .../hdfs/web/TestByteRangeInputStream.java | 79 + .../hdfs/web/TestFSMainOperationsWebHdfs.java | 4 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 4 +- .../hadoop/hdfs/web/TestWebHDFSOAuth2.java | 216 ++ .../web/TestWebHdfsWithMultipleNameNodes.java | 3 +- .../hdfs/web/oauth2/TestAccessTokenTimer.java | 63 + ...ClientCredentialTimeBasedTokenRefresher.java | 138 + ...TestRefreshTokenTimeBasedTokenRefresher.java | 138 + .../hadoop/tools/TestHdfsConfigFields.java | 6 +- .../org/apache/hadoop/tools/TestJMXGet.java | 3 + .../apache/hadoop/tracing/TestTraceAdmin.java | 17 +- .../org/apache/hadoop/tracing/TestTracing.java | 86 +- .../TestTracingShortCircuitLocalRead.java | 18 +- .../src/test/resources/hadoop-252-dfs-dir.tgz | Bin 0 -> 14112 bytes .../test/resources/hadoop-metrics2.properties | 85 + .../src/test/resources/log4j.properties | 13 + .../src/test/resources/testDeleteConf.xml | 83 + .../src/test/resources/testHDFSConf.xml | 198 +- hadoop-mapreduce-project/CHANGES.txt | 125 +- hadoop-mapreduce-project/bin/mapred | 8 + .../v2/app/job/impl/TaskAttemptImpl.java | 94 +- .../v2/app/rm/RMContainerAllocator.java | 12 +- .../v2/app/rm/RMContainerRequestor.java | 2 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 11 +- .../v2/app/job/impl/TestTaskAttempt.java | 162 + .../v2/app/launcher/TestContainerLauncher.java | 11 + .../app/launcher/TestContainerLauncherImpl.java | 9 + .../app/local/TestLocalContainerAllocator.java | 6 +- .../v2/app/rm/TestRMContainerAllocator.java | 126 +- .../apache/hadoop/mapred/LocalJobRunner.java | 27 + .../java/org/apache/hadoop/mapred/Master.java | 27 +- .../java/org/apache/hadoop/mapred/TaskLog.java | 4 +- .../org/apache/hadoop/mapreduce/Cluster.java | 2 +- .../lib/input/UncompressedSplitLineReader.java | 31 +- .../lib/output/FileOutputCommitter.java | 52 +- .../hadoop/mapred/TestLineRecordReader.java | 138 + .../org/apache/hadoop/mapred/TestMaster.java | 13 + .../lib/input/TestLineRecordReader.java | 161 + .../org/apache/hadoop/mapred/YARNRunner.java | 4 + .../org/apache/hadoop/cli/CLITestCmdMR.java | 3 +- .../org/apache/hadoop/fs/TestFileSystem.java | 14 +- .../java/org/apache/hadoop/hdfs/NNBench.java | 14 +- .../hadoop/mapred/TestLocalJobSubmission.java | 25 + .../apache/hadoop/mapred/TestYARNRunner.java | 10 +- .../mapreduce/security/TestMRCredentials.java | 4 +- .../apache/hadoop/mapred/ShuffleHandler.java | 177 +- .../hadoop/mapred/TestShuffleHandler.java | 129 + hadoop-project/pom.xml | 17 +- hadoop-project/src/site/site.xml | 1 + .../dev-support/findbugs-exclude.xml | 32 + hadoop-tools/hadoop-archive-logs/pom.xml | 189 ++ .../apache/hadoop/tools/HadoopArchiveLogs.java | 552 +++ .../hadoop/tools/HadoopArchiveLogsRunner.java | 180 + .../hadoop/tools/TestHadoopArchiveLogs.java | 350 ++ .../tools/TestHadoopArchiveLogsRunner.java | 143 + .../org/apache/hadoop/fs/s3a/Constants.java | 4 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 3 + .../src/site/markdown/tools/hadoop-aws/index.md | 4 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 3 - .../fs/azure/NativeAzureFileSystemBaseTest.java | 1 - .../tools/mapred/UniformSizeInputFormat.java | 3 +- .../hadoop/tools/util/ProducerConsumer.java | 19 +- .../apache/hadoop/tools/util/WorkReport.java | 2 +- .../apache/hadoop/tools/util/WorkRequest.java | 2 +- .../src/site/markdown/DistCp.md.vm | 5 +- .../hadoop/mapred/gridmix/SerialJobFactory.java | 2 +- .../hadoop/yarn/sls/RumenToSLSConverter.java | 2 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 14 + .../yarn/sls/scheduler/RMNodeWrapper.java | 13 + .../sls/scheduler/ResourceSchedulerWrapper.java | 21 +- .../sls/scheduler/SLSCapacityScheduler.java | 19 +- .../org/apache/hadoop/streaming/UtilTest.java | 5 - hadoop-tools/hadoop-tools-dist/pom.xml | 5 + hadoop-tools/pom.xml | 1 + hadoop-yarn-project/CHANGES.txt | 603 ++-- .../yarn/api/ContainerManagementProtocol.java | 30 +- .../api/protocolrecords/AllocateRequest.java | 57 +- .../api/protocolrecords/AllocateResponse.java | 42 +- .../IncreaseContainersResourceRequest.java | 75 + .../IncreaseContainersResourceResponse.java | 93 + .../MoveApplicationAcrossQueuesResponse.java | 5 +- .../yarn/api/records/ApplicationReport.java | 31 +- .../records/ContainerResourceChangeRequest.java | 117 + .../api/records/ContainerResourceDecrease.java | 78 - .../api/records/ContainerResourceIncrease.java | 84 - .../ContainerResourceIncreaseRequest.java | 80 - .../yarn/api/records/ContainerStatus.java | 13 + .../hadoop/yarn/api/records/NodeLabel.java | 16 + .../hadoop/yarn/api/records/ResourceOption.java | 3 + .../hadoop/yarn/conf/YarnConfiguration.java | 37 +- .../hadoop/yarn/server/api/ContainerType.java | 1 - .../ResourceManagerAdministrationProtocol.java | 24 +- .../RefreshClusterMaxPriorityRequest.java | 35 + .../RefreshClusterMaxPriorityResponse.java | 36 + .../RefreshNodesResourcesRequest.java | 39 + .../RefreshNodesResourcesResponse.java | 39 + .../proto/containermanagement_protocol.proto | 1 + ...esourcemanager_administration_protocol.proto | 4 +- ..._server_resourcemanager_service_protos.proto | 11 + .../src/main/proto/yarn_protos.proto | 16 +- .../src/main/proto/yarn_service_protos.proto | 16 +- .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../hadoop-yarn/hadoop-yarn-client/pom.xml | 12 + .../yarn/client/api/impl/AMRMClientImpl.java | 2 +- .../impl/ContainerManagementProtocolProxy.java | 14 +- .../yarn/client/api/impl/YarnClientImpl.java | 7 +- .../hadoop/yarn/client/cli/ApplicationCLI.java | 10 +- .../apache/hadoop/yarn/client/cli/LogsCLI.java | 104 +- .../apache/hadoop/yarn/client/cli/QueueCLI.java | 11 +- .../hadoop/yarn/client/cli/RMAdminCLI.java | 76 +- .../hadoop/yarn/client/TestRMFailover.java | 27 + .../yarn/client/TestResourceTrackerOnHA.java | 2 +- .../api/impl/TestAMRMClientOnRMRestart.java | 8 +- .../hadoop/yarn/client/cli/TestLogsCLI.java | 53 +- .../hadoop/yarn/client/cli/TestRMAdminCLI.java | 40 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 8 +- .../application_1440536969523_0001.har/_SUCCESS | 0 .../application_1440536969523_0001.har/_index | 3 + .../_masterindex | 2 + .../application_1440536969523_0001.har/part-0 | Bin 0 -> 795 bytes .../hadoop-yarn/hadoop-yarn-common/pom.xml | 4 + ...ContainerManagementProtocolPBClientImpl.java | 20 + ...ontainerManagementProtocolPBServiceImpl.java | 22 + .../impl/pb/AllocateRequestPBImpl.java | 119 +- .../impl/pb/AllocateResponsePBImpl.java | 175 +- ...IncreaseContainersResourceRequestPBImpl.java | 170 + ...ncreaseContainersResourceResponsePBImpl.java | 241 ++ .../impl/pb/ApplicationReportPBImpl.java | 38 + .../ContainerResourceChangeRequestPBImpl.java | 141 + .../pb/ContainerResourceDecreasePBImpl.java | 136 - .../pb/ContainerResourceIncreasePBImpl.java | 171 - .../ContainerResourceIncreaseRequestPBImpl.java | 141 - .../records/impl/pb/ContainerStatusPBImpl.java | 31 +- .../org/apache/hadoop/yarn/client/RMProxy.java | 3 +- .../apache/hadoop/yarn/client/ServerProxy.java | 3 +- .../yarn/logaggregation/LogCLIHelpers.java | 16 +- .../nodelabels/CommonNodeLabelsManager.java | 2 +- .../yarn/security/ContainerTokenSelector.java | 2 +- .../hadoop/yarn/security/NMTokenSelector.java | 2 +- ...nagerAdministrationProtocolPBClientImpl.java | 39 + ...agerAdministrationProtocolPBServiceImpl.java | 45 + .../RefreshClusterMaxPriorityRequestPBImpl.java | 74 + ...RefreshClusterMaxPriorityResponsePBImpl.java | 73 + .../pb/RefreshNodesResourcesRequestPBImpl.java | 72 + .../pb/RefreshNodesResourcesResponsePBImpl.java | 72 + .../resource/DefaultResourceCalculator.java | 5 + .../resource/DominantResourceCalculator.java | 6 + .../yarn/util/resource/ResourceCalculator.java | 5 + .../hadoop/yarn/util/resource/Resources.java | 5 + .../hadoop/yarn/webapp/YarnWebParams.java | 1 + .../yarn/webapp/log/AggregatedLogsBlock.java | 7 + .../src/main/resources/yarn-default.xml | 63 +- .../hadoop/yarn/TestContainerLaunchRPC.java | 8 + .../yarn/TestContainerResourceIncreaseRPC.java | 162 + .../java/org/apache/hadoop/yarn/TestRPC.java | 8 + .../hadoop/yarn/api/TestAllocateRequest.java | 73 - .../hadoop/yarn/api/TestAllocateResponse.java | 114 - .../hadoop/yarn/api/TestApplicatonReport.java | 2 +- .../yarn/api/TestContainerResourceDecrease.java | 66 - .../yarn/api/TestContainerResourceIncrease.java | 74 - .../TestContainerResourceIncreaseRequest.java | 68 - .../hadoop/yarn/api/TestPBImplRecords.java | 70 +- .../hadoop/yarn/event/TestAsyncDispatcher.java | 2 + .../logaggregation/TestAggregatedLogsBlock.java | 63 +- .../util/resource/TestResourceCalculator.java | 30 +- .../hadoop/yarn/webapp/JerseyTestBase.java | 14 +- .../application_1440536969523_0001.har/_SUCCESS | 0 .../application_1440536969523_0001.har/_index | 3 + .../_masterindex | 2 + .../application_1440536969523_0001.har/part-0 | Bin 0 -> 795 bytes .../hadoop-yarn/hadoop-yarn-registry/pom.xml | 6 +- ...pplicationHistoryManagerOnTimelineStore.java | 41 +- .../ApplicationHistoryServer.java | 13 +- ...pplicationHistoryManagerOnTimelineStore.java | 43 +- .../TestApplicationHistoryServer.java | 4 +- .../protocolrecords/NodeHeartbeatResponse.java | 5 + .../impl/pb/NodeHeartbeatResponsePBImpl.java | 77 +- .../yarn/server/api/records/NodeStatus.java | 17 +- .../api/records/impl/pb/NodeStatusPBImpl.java | 83 +- .../metrics/ApplicationMetricsConstants.java | 9 + .../hadoop/yarn/server/utils/BuilderUtils.java | 4 +- .../server/utils/YarnServerSecurityUtils.java | 142 + .../hadoop/yarn/server/webapp/AppBlock.java | 6 + .../hadoop/yarn/server/webapp/WebPageUtils.java | 3 +- .../hadoop/yarn/server/webapp/WebServices.java | 9 + .../hadoop/yarn/server/webapp/dao/AppInfo.java | 12 + .../main/proto/yarn_server_common_protos.proto | 3 +- .../yarn_server_common_service_protos.proto | 1 + .../hadoop/yarn/TestYarnServerApiClasses.java | 39 +- .../CMgrDecreaseContainersResourceEvent.java | 37 + .../nodemanager/ContainerManagerEventType.java | 1 + .../hadoop/yarn/server/nodemanager/Context.java | 3 + .../nodemanager/LinuxContainerExecutor.java | 2 +- .../nodemanager/LocalDirsHandlerService.java | 33 +- .../yarn/server/nodemanager/NodeManager.java | 12 +- .../nodemanager/NodeStatusUpdaterImpl.java | 84 +- .../amrmproxy/AMRMProxyApplicationContext.java | 70 + .../AMRMProxyApplicationContextImpl.java | 132 + .../nodemanager/amrmproxy/AMRMProxyService.java | 592 ++++ .../amrmproxy/AMRMProxyTokenSecretManager.java | 265 ++ .../amrmproxy/AbstractRequestInterceptor.java | 102 + .../amrmproxy/DefaultRequestInterceptor.java | 138 + .../amrmproxy/RequestInterceptor.java | 71 + .../containermanager/ContainerManagerImpl.java | 351 +- .../containermanager/container/Container.java | 2 + .../container/ContainerImpl.java | 24 +- .../localizer/LocalResourcesTrackerImpl.java | 64 +- .../localizer/ResourceLocalizationService.java | 2 +- .../logaggregation/AppLogAggregator.java | 2 + .../logaggregation/AppLogAggregatorImpl.java | 6 + .../logaggregation/LogAggregationService.java | 25 +- .../ChangeMonitoringContainerResourceEvent.java | 37 + .../monitor/ContainersMonitorEventType.java | 3 +- .../monitor/ContainersMonitorImpl.java | 207 +- .../nodemanager/metrics/NodeManagerMetrics.java | 11 + .../nodelabels/AbstractNodeLabelsProvider.java | 7 +- .../ConfigurationNodeLabelsProvider.java | 11 - .../recovery/NMLeveldbStateStoreService.java | 22 + .../recovery/NMNullStateStoreService.java | 6 + .../recovery/NMStateStoreService.java | 15 + .../nodemanager/webapp/NMWebServices.java | 12 +- .../nodemanager/webapp/dao/ContainerInfo.java | 46 +- .../nodemanager/DummyContainerManager.java | 13 +- .../TestContainerManagerWithLCE.java | 33 + .../yarn/server/nodemanager/TestEventFlow.java | 3 +- .../TestLocalDirsHandlerService.java | 18 + .../nodemanager/TestNodeManagerResync.java | 260 +- .../nodemanager/TestNodeStatusUpdater.java | 8 +- .../amrmproxy/BaseAMRMProxyTest.java | 682 ++++ .../amrmproxy/MockRequestInterceptor.java | 65 + .../amrmproxy/MockResourceManagerFacade.java | 467 +++ .../PassThroughRequestInterceptor.java | 58 + .../amrmproxy/TestAMRMProxyService.java | 484 +++ .../BaseContainerManagerTest.java | 46 +- .../containermanager/TestContainerManager.java | 293 +- .../TestContainerManagerRecovery.java | 236 +- .../containermanager/TestNMProxy.java | 3 +- .../TestLocalResourcesTrackerImpl.java | 86 +- .../localizer/TestResourceRetention.java | 2 +- .../TestLogAggregationService.java | 36 +- .../monitor/MockResourceCalculatorPlugin.java | 69 + .../MockResourceCalculatorProcessTree.java | 57 + .../TestContainersMonitorResourceChange.java | 248 ++ .../metrics/TestNodeManagerMetrics.java | 18 +- .../TestConfigurationNodeLabelsProvider.java | 69 +- .../recovery/NMMemoryStateStoreService.java | 11 +- .../TestNMLeveldbStateStoreService.java | 11 + .../nodemanager/webapp/MockContainer.java | 4 + .../webapp/TestNMWebServicesContainers.java | 13 +- .../server/resourcemanager/AdminService.java | 109 +- .../ApplicationMasterService.java | 91 +- .../server/resourcemanager/ClientRMService.java | 35 +- .../resourcemanager/EmbeddedElectorService.java | 4 + .../resourcemanager/NodesListManager.java | 144 +- .../resourcemanager/RMActiveServiceContext.java | 16 +- .../server/resourcemanager/RMAppManager.java | 9 + .../server/resourcemanager/RMAuditLogger.java | 2 + .../yarn/server/resourcemanager/RMContext.java | 5 + .../server/resourcemanager/RMContextImpl.java | 12 +- .../resourcemanager/RMFatalEventType.java | 5 +- .../server/resourcemanager/RMServerUtils.java | 164 + .../server/resourcemanager/ResourceManager.java | 4 +- .../resourcemanager/ResourceTrackerService.java | 27 +- .../resourcemanager/amlauncher/AMLauncher.java | 27 +- .../blacklist/BlacklistManager.java | 47 + .../blacklist/BlacklistUpdates.java | 47 + .../blacklist/DisabledBlacklistManager.java | 45 + .../blacklist/SimpleBlacklistManager.java | 84 + .../metrics/ApplicationCreatedEvent.java | 16 +- .../metrics/ApplicationUpdatedEvent.java | 54 + .../metrics/SystemMetricsEventType.java | 1 + .../metrics/SystemMetricsPublisher.java | 43 +- .../placement/PlacementManager.java | 95 + .../placement/PlacementRule.java | 55 + .../UserGroupMappingPlacementRule.java | 164 + .../resourcemanager/recovery/RMStateStore.java | 15 +- .../resource/DynamicResourceConfiguration.java | 149 + .../server/resourcemanager/rmapp/RMApp.java | 6 + .../server/resourcemanager/rmapp/RMAppImpl.java | 64 +- .../rmapp/attempt/RMAppAttempt.java | 7 + .../rmapp/attempt/RMAppAttemptImpl.java | 60 +- .../rmcontainer/RMContainer.java | 4 + .../RMContainerChangeResourceEvent.java | 44 + .../rmcontainer/RMContainerEventType.java | 13 +- .../rmcontainer/RMContainerImpl.java | 121 +- .../RMContainerUpdatesAcquiredEvent.java | 35 + .../server/resourcemanager/rmnode/RMNode.java | 12 +- .../rmnode/RMNodeDecreaseContainerEvent.java | 39 + .../resourcemanager/rmnode/RMNodeEventType.java | 3 +- .../resourcemanager/rmnode/RMNodeImpl.java | 549 ++- .../rmnode/RMNodeStatusEvent.java | 33 +- .../scheduler/AbstractYarnScheduler.java | 177 +- .../resourcemanager/scheduler/Allocation.java | 22 +- .../scheduler/AppSchedulingInfo.java | 327 +- .../resourcemanager/scheduler/QueueMetrics.java | 16 +- .../scheduler/SchedContainerChangeRequest.java | 118 + .../scheduler/SchedulerApplication.java | 2 +- .../scheduler/SchedulerApplicationAttempt.java | 281 +- .../scheduler/SchedulerNode.java | 31 + .../scheduler/SchedulerUtils.java | 11 +- .../scheduler/YarnScheduler.java | 36 +- .../scheduler/capacity/AbstractCSQueue.java | 50 +- .../scheduler/capacity/CSAssignment.java | 9 + .../scheduler/capacity/CSQueue.java | 42 + .../scheduler/capacity/CapacityScheduler.java | 289 +- .../CapacitySchedulerConfiguration.java | 32 +- .../scheduler/capacity/LeafQueue.java | 149 +- .../scheduler/capacity/ParentQueue.java | 115 +- .../allocator/AbstractContainerAllocator.java | 131 + .../capacity/allocator/ContainerAllocator.java | 149 +- .../allocator/IncreaseContainerAllocator.java | 365 ++ .../allocator/RegularContainerAllocator.java | 30 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 77 +- .../common/fica/FiCaSchedulerUtils.java | 48 - .../scheduler/fair/FSAppAttempt.java | 19 +- .../scheduler/fair/FSLeafQueue.java | 5 +- .../scheduler/fair/FSParentQueue.java | 3 +- .../scheduler/fair/FairScheduler.java | 70 +- .../fair/FairSchedulerConfiguration.java | 22 +- .../scheduler/fair/QueueManager.java | 104 +- .../scheduler/fifo/FifoScheduler.java | 36 +- .../scheduler/policy/OrderingPolicy.java | 2 +- .../resourcemanager/webapp/AboutBlock.java | 2 + .../webapp/CapacitySchedulerPage.java | 17 +- .../webapp/JAXBContextResolver.java | 2 +- .../webapp/MetricsOverviewTable.java | 3 + .../resourcemanager/webapp/NodeLabelsPage.java | 6 +- .../resourcemanager/webapp/RMAppBlock.java | 3 +- .../server/resourcemanager/webapp/RMWebApp.java | 5 + .../resourcemanager/webapp/RMWebAppFilter.java | 93 +- .../resourcemanager/webapp/RMWebServices.java | 123 +- .../webapp/dao/AppAttemptInfo.java | 10 +- .../resourcemanager/webapp/dao/AppInfo.java | 29 +- .../resourcemanager/webapp/dao/AppPriority.java | 44 + .../dao/CapacitySchedulerLeafQueueInfo.java | 14 +- .../resourcemanager/webapp/dao/ClusterInfo.java | 6 + .../webapp/dao/SchedulerInfo.java | 6 + .../server/resourcemanager/Application.java | 2 +- .../yarn/server/resourcemanager/MockAM.java | 9 + .../yarn/server/resourcemanager/MockNM.java | 2 +- .../yarn/server/resourcemanager/MockNodes.java | 13 + .../yarn/server/resourcemanager/MockRM.java | 45 +- .../server/resourcemanager/NodeManager.java | 13 +- .../server/resourcemanager/RMHATestBase.java | 2 +- .../resourcemanager/TestAMAuthorization.java | 8 + .../server/resourcemanager/TestAppManager.java | 54 +- .../resourcemanager/TestApplicationCleanup.java | 6 +- .../TestApplicationMasterLauncher.java | 75 + .../TestApplicationMasterService.java | 144 +- .../resourcemanager/TestClientRMService.java | 343 +- .../resourcemanager/TestRMAdminService.java | 80 + .../yarn/server/resourcemanager/TestRMHA.java | 71 + .../resourcemanager/TestRMNodeTransitions.java | 202 +- .../TestTokenClientRMService.java | 300 ++ .../TestWorkPreservingRMRestart.java | 21 +- .../applicationsmanager/MockAsm.java | 10 + .../applicationsmanager/TestAMRestart.java | 162 +- .../blacklist/TestBlacklistManager.java | 118 + .../TestRMAppLogAggregationStatus.java | 12 +- .../metrics/TestSystemMetricsPublisher.java | 68 +- .../TestUserGroupMappingPlacementRule.java | 89 + .../server/resourcemanager/rmapp/MockRMApp.java | 13 + .../rmapp/TestNodesListManager.java | 102 + .../rmapp/TestRMAppTransitions.java | 9 +- .../attempt/TestRMAppAttemptTransitions.java | 53 +- .../rmcontainer/TestRMContainerImpl.java | 117 +- .../scheduler/TestAbstractYarnScheduler.java | 14 +- .../capacity/TestCapacityScheduler.java | 277 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 249 +- .../scheduler/capacity/TestChildQueueOrder.java | 4 +- .../capacity/TestContainerAllocation.java | 50 +- .../capacity/TestContainerResizing.java | 963 ++++++ .../scheduler/capacity/TestLeafQueue.java | 4 +- .../scheduler/capacity/TestParentQueue.java | 4 +- .../scheduler/capacity/TestQueueMappings.java | 203 +- .../scheduler/capacity/TestReservations.java | 9 +- .../scheduler/fair/FairSchedulerTestBase.java | 14 +- .../fair/TestContinuousScheduling.java | 2 +- .../scheduler/fair/TestFSParentQueue.java | 79 + .../scheduler/fair/TestFairScheduler.java | 150 +- .../scheduler/fifo/TestFifoScheduler.java | 34 +- .../security/TestAMRMTokens.java | 3 +- .../security/TestRMDelegationTokens.java | 3 + .../resourcemanager/webapp/TestNodesPage.java | 5 +- .../webapp/TestRMWebServices.java | 8 +- .../webapp/TestRMWebServicesApps.java | 34 +- .../TestRMWebServicesAppsModification.java | 120 + .../webapp/TestRMWebServicesCapacitySched.java | 2 +- .../TestRMWebServicesDelegationTokens.java | 6 +- .../hadoop/yarn/server/MiniYARNCluster.java | 42 +- .../yarn/server/webproxy/AppReportFetcher.java | 49 +- .../server/webproxy/WebAppProxyServlet.java | 46 +- .../server/webproxy/TestWebAppProxyServlet.java | 113 +- .../src/site/markdown/CapacityScheduler.md | 6 +- .../src/site/markdown/NodeLabel.md | 140 + .../src/site/markdown/ResourceManagerRest.md | 152 +- .../src/site/markdown/TimelineServer.md | 38 +- 989 files changed, 60801 insertions(+), 36348 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 2df5955,0e2d541..88b4594 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@@ -41,7 -41,7 +41,7 @@@ message ExtendedBlockProto required uint64 blockId = 2; // the local id within a pool required uint64 generationStamp = 3; optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid -- // here for historical reasons ++ // here for historical reasons } /** @@@ -51,9 -51,9 +51,9 @@@ message DatanodeIDProto required string ipAddr = 1; // IP address required string hostName = 2; // hostname required string datanodeUuid = 3; // UUID assigned to the Datanode. For -- // upgraded clusters this is the same -- // as the original StorageID of the -- // Datanode. ++ // upgraded clusters this is the same ++ // as the original StorageID of the ++ // Datanode. required uint32 xferPort = 4; // data streaming port required uint32 infoPort = 5; // datanode http port required uint32 ipcPort = 6; // ipc server port @@@ -156,8 -157,8 +157,8 @@@ message StorageTypeQuotaInfoProto * */ message CorruptFileBlocksProto { -- repeated string files = 1; -- required string cookie = 2; ++ repeated string files = 1; ++ required string cookie = 2; } /** @@@ -201,20 -202,14 +202,14 @@@ message BlockStoragePolicyProto /** * A LocatedBlock gives information about a block and its location. -- */ ++ */ message LocatedBlockProto { required ExtendedBlockProto b = 1; required uint64 offset = 2; // offset of first byte of block in the file repeated DatanodeInfoProto locs = 3; // Locations ordered by proximity to client ip required bool corrupt = 4; // true if all replicas of a block are corrupt, else false -- // If block has few corrupt replicas, they are filtered and -- // their locations are not part of this object ++ // If block has few corrupt replicas, they are filtered and ++ // their locations are not part of this object required hadoop.common.TokenProto blockToken = 5; repeated bool isCached = 6 [packed=true]; // if a location in locs is cached @@@ -235,16 -230,16 +230,16 @@@ message DataEncryptionKeyProto * Cipher suite. */ enum CipherSuiteProto { -- UNKNOWN = 1; -- AES_CTR_NOPADDING = 2; ++ UNKNOWN = 1; ++ AES_CTR_NOPADDING = 2; } /** * Crypto protocol version used to access encrypted files. */ enum CryptoProtocolVersionProto { -- UNKNOWN_PROTOCOL_VERSION = 1; -- ENCRYPTION_ZONES = 2; ++ UNKNOWN_PROTOCOL_VERSION = 1; ++ ENCRYPTION_ZONES = 2; } /** @@@ -336,7 -331,7 +331,7 @@@ message HdfsFileStatusProto optional FileEncryptionInfoProto fileEncryptionInfo = 15; optional uint32 storagePolicy = 16 [default = 0]; // block storage policy id --} ++} /** * Checksum algorithms/types used in HDFS http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java index 230786c,9189c32..ad1aa3b --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java @@@ -17,10 -17,10 +17,11 @@@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.Iterator; - +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; import org.apache.hadoop.hdfs.protocol.Block; + import org.apache.hadoop.hdfs.server.namenode.INodeId; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.util.GSet; import org.apache.hadoop.util.LightWeightGSet; http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index fd1c0f0,0b398c5..0b143f3 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@@ -17,8 -17,22 +17,9 @@@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; - import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; + import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index 8946256,2fe67fd..6bd556f --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@@ -148,15 -152,12 +154,16 @@@ import org.apache.hadoop.hdfs.security. import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; + import org.apache.hadoop.hdfs.server.common.MetricsLoggerTask; import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.StorageInfo; +import org.apache.hadoop.hdfs.server.datanode.DataStorage.VolumeBuilder; import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources; +import org.apache.hadoop.hdfs.server.datanode.dataset.DatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; +import org.apache.hadoop.hdfs.server.datanode.dataset.VolumeSpi; import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; import org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer; import org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock; @@@ -630,7 -602,7 +668,6 @@@ public class DataNode extends Reconfigu private synchronized void refreshVolumes(String newVolumes) throws IOException { Configuration conf = getConf(); conf.set(DFS_DATANODE_DATA_DIR_KEY, newVolumes); - - ExecutorService service = null; int numOldDataDirs = dataDirs.size(); ChangedVolumes changedVolumes = parseChangedVolumes(newVolumes); StringBuilder errorMessageBuilder = new StringBuilder(); @@@ -2233,11 -2176,10 +2284,11 @@@ unbufIn = saslStreams.in; out = new DataOutputStream(new BufferedOutputStream(unbufOut, - DFSUtil.getSmallBufferSize(conf))); + DFSUtilClient.getSmallBufferSize(conf))); in = new DataInputStream(unbufIn); blockSender = new BlockSender(b, 0, b.getNumBytes(), - false, false, true, DataNode.this, null, cachingStrategy); + false, false, true, DataNode.this, data, + null, cachingStrategy); DatanodeInfo srcNode = new DatanodeInfo(bpReg); new Sender(out).writeBlock(b, targetStorageTypes[0], accessToken, http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java index ff537b8,b8ea5bf..1663116 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java @@@ -310,17 -401,44 +402,50 @@@ public class DirectoryScanner implement } } + /** + * Create a new directory scanner, but don't cycle it running yet. + * + * @param datanode the parent datanode + * @param dataset the dataset to scan + * @param conf the Configuration object + */ - DirectoryScanner(DataNode datanode, FsDatasetSpi<?> dataset, Configuration conf) { + DirectoryScanner(DataNode datanode, DatasetSpi<?> dataset, + Configuration conf) { + if (!(dataset instanceof FsDatasetSpi)) { + throw new IllegalArgumentException( + "DirectoryScanner not implemented for " + dataset.getClass()); + } this.datanode = datanode; - this.dataset = dataset; + this.dataset = (FsDatasetSpi<?>) dataset; int interval = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY, DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT); - scanPeriodMsecs = interval * 1000L; //msec + scanPeriodMsecs = interval * MILLIS_PER_SECOND; //msec + + int throttle = + conf.getInt( + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY, + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_DEFAULT); + + if ((throttle > MILLIS_PER_SECOND) || (throttle <= 0)) { + if (throttle > MILLIS_PER_SECOND) { ++ + LOG.error( + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY + + " set to value above 1000 ms/sec. Assuming default value of " + + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_DEFAULT); + } else { + LOG.error( + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_KEY + + " set to value below 1 ms/sec. Assuming default value of " + + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_DEFAULT); + } + + throttleLimitMsPerSec = + DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THROTTLE_LIMIT_MS_PER_SEC_DEFAULT; + } else { + throttleLimitMsPerSec = throttle; + } + int threads = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_KEY, DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_THREADS_DEFAULT); http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java index d58ce79,9e16121..b2ef246 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsVolumeSpi.java @@@ -45,14 -41,34 +45,14 @@@ public interface FsVolumeSpi extends Vo /** @return the directory for the finalized blocks in the block pool. */ File getFinalizedDir(String bpid) throws IOException; - StorageType getStorageType(); - - /** Returns true if the volume is NOT backed by persistent storage. */ - boolean isTransientStorage(); - /** - * Reserve disk space for an RBW block so a writer does not run out of - * space before the block is full. + * Reserve disk space for a block (RBW or Re-replicating) + * so a writer does not run out of space before the block is full. */ - void reserveSpaceForRbw(long bytesToReserve); + void reserveSpaceForReplica(long bytesToReserve); /** - * Release disk space previously reserved for RBW block. + * Release disk space previously reserved for block opened for write. */ void releaseReservedSpace(long bytesToRelease); http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d313d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java index bbbf5f5,0000000..3147767 mode 100644,000000..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerManager.java @@@ -1,322 -1,0 +1,323 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.storagecontainer; + +import com.google.protobuf.BlockingService; +import org.apache.hadoop.ha.HAServiceProtocol; +import org.apache.hadoop.hdfs.DFSUtil; ++import org.apache.hadoop.hdfs.DFSUtilClient; +import org.apache.hadoop.hdfs.protocol.*; +import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolServerSideTranslatorPB; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager; +import org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap; +import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.Namesystem; +import org.apache.hadoop.hdfs.server.protocol.*; +import org.apache.hadoop.ipc.ProtobufRpcEngine; +import org.apache.hadoop.ipc.RPC; +import org.apache.hadoop.ipc.WritableRpcEngine; +import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.storagecontainer.protocol.ContainerLocationProtocol; +import org.apache.hadoop.util.LightWeightGSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.List; + +import static org.apache.hadoop.hdfs.DFSConfigKeys.*; + +/** + * Service that allocates storage containers and tracks their + * location. + */ +public class StorageContainerManager + implements DatanodeProtocol, ContainerLocationProtocol { + + public static final Logger LOG = + LoggerFactory.getLogger(StorageContainerManager.class); + + private final Namesystem ns = new StorageContainerNameService(); + private final BlockManager blockManager; + + private long txnId = 234; + + /** The RPC server that listens to requests from DataNodes. */ + private final RPC.Server serviceRpcServer; + private final InetSocketAddress serviceRPCAddress; + + /** The RPC server that listens to requests from clients. */ + private final RPC.Server clientRpcServer; + private final InetSocketAddress clientRpcAddress; + + public StorageContainerManager(OzoneConfiguration conf) + throws IOException { + BlocksMap containerMap = new BlocksMap( + LightWeightGSet.computeCapacity(2.0, "BlocksMap"), + new StorageContainerMap()); + this.blockManager = new BlockManager(ns, conf, containerMap); + + int handlerCount = + conf.getInt(DFS_NAMENODE_HANDLER_COUNT_KEY, + DFS_NAMENODE_HANDLER_COUNT_DEFAULT); + + RPC.setProtocolEngine(conf, DatanodeProtocolPB.class, + ProtobufRpcEngine.class); + + DatanodeProtocolServerSideTranslatorPB dnProtoPbTranslator = + new DatanodeProtocolServerSideTranslatorPB(this); + BlockingService dnProtoPbService = + DatanodeProtocolProtos.DatanodeProtocolService + .newReflectiveBlockingService(dnProtoPbTranslator); + + WritableRpcEngine.ensureInitialized(); + + InetSocketAddress serviceRpcAddr = NameNode.getServiceAddress(conf, false); + if (serviceRpcAddr != null) { + String bindHost = + conf.getTrimmed(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY); + if (bindHost == null || bindHost.isEmpty()) { + bindHost = serviceRpcAddr.getHostName(); + } + LOG.info("Service RPC server is binding to " + bindHost + ":" + + serviceRpcAddr.getPort()); + + int serviceHandlerCount = + conf.getInt(DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY, + DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT); + serviceRpcServer = new RPC.Builder(conf) + .setProtocol( + org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB.class) + .setInstance(dnProtoPbService) + .setBindAddress(bindHost) + .setPort(serviceRpcAddr.getPort()) + .setNumHandlers(serviceHandlerCount) + .setVerbose(false) + .setSecretManager(null) + .build(); + + DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, + serviceRpcServer); + + InetSocketAddress listenAddr = serviceRpcServer.getListenerAddress(); + serviceRPCAddress = new InetSocketAddress( + serviceRpcAddr.getHostName(), listenAddr.getPort()); + conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, + NetUtils.getHostPortString(serviceRPCAddress)); + } else { + serviceRpcServer = null; + serviceRPCAddress = null; + } + - InetSocketAddress rpcAddr = NameNode.getAddress(conf); ++ InetSocketAddress rpcAddr = DFSUtilClient.getNNAddress(conf); + String bindHost = conf.getTrimmed(DFS_NAMENODE_RPC_BIND_HOST_KEY); + if (bindHost == null || bindHost.isEmpty()) { + bindHost = rpcAddr.getHostName(); + } + LOG.info("RPC server is binding to " + bindHost + ":" + rpcAddr.getPort()); + + clientRpcServer = new RPC.Builder(conf) + .setProtocol( + org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolPB.class) + .setInstance(dnProtoPbService) + .setBindAddress(bindHost) + .setPort(rpcAddr.getPort()) + .setNumHandlers(handlerCount) + .setVerbose(false) + .setSecretManager(null) + .build(); + + DFSUtil.addPBProtocol(conf, DatanodeProtocolPB.class, dnProtoPbService, + clientRpcServer); + + // The rpc-server port can be ephemeral... ensure we have the correct info + InetSocketAddress listenAddr = clientRpcServer.getListenerAddress(); + clientRpcAddress = new InetSocketAddress( + rpcAddr.getHostName(), listenAddr.getPort()); + conf.set(FS_DEFAULT_NAME_KEY, - NameNode.getUri(clientRpcAddress).toString()); ++ DFSUtilClient.getNNUri(clientRpcAddress).toString()); + } + + @Override + public DatanodeRegistration registerDatanode( + DatanodeRegistration registration) throws IOException { + ns.writeLock(); + try { + blockManager.getDatanodeManager().registerDatanode(registration); + } finally { + ns.writeUnlock(); + } + return registration; + } + + @Override + public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration, + StorageReport[] reports, long dnCacheCapacity, long dnCacheUsed, + int xmitsInProgress, int xceiverCount, int failedVolumes, + VolumeFailureSummary volumeFailureSummary, + boolean requestFullBlockReportLease) throws IOException { + ns.readLock(); + try { + final int maxTransfer = blockManager.getMaxReplicationStreams() + - xmitsInProgress; + DatanodeCommand[] cmds = blockManager.getDatanodeManager() + .handleHeartbeat(registration, reports, ns.getBlockPoolId(), 0, 0, + xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary); + + return new HeartbeatResponse(cmds, + new NNHAStatusHeartbeat(HAServiceProtocol.HAServiceState.ACTIVE, + txnId), null, 0); + } finally { + ns.readUnlock(); + } + } + + @Override + public DatanodeCommand blockReport(DatanodeRegistration registration, + String poolId, StorageBlockReport[] reports, + BlockReportContext context) throws IOException { + for (int r = 0; r < reports.length; r++) { + final BlockListAsLongs storageContainerList = reports[r].getBlocks(); + blockManager.processReport(registration, reports[r].getStorage(), + storageContainerList, context, (r == reports.length - 1)); + } + return null; + } + + @Override + public DatanodeCommand cacheReport(DatanodeRegistration registration, + String poolId, List<Long> blockIds) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void blockReceivedAndDeleted(DatanodeRegistration registration, + String poolId, StorageReceivedDeletedBlocks[] rcvdAndDeletedBlocks) + throws IOException { + for(StorageReceivedDeletedBlocks r : rcvdAndDeletedBlocks) { + ns.writeLock(); + try { + blockManager.processIncrementalBlockReport(registration, r); + } finally { + ns.writeUnlock(); + } + } + } + + @Override + public void errorReport(DatanodeRegistration registration, + int errorCode, String msg) throws IOException { + String dnName = + (registration == null) ? "Unknown DataNode" : registration.toString(); + + if (errorCode == DatanodeProtocol.NOTIFY) { + LOG.info("Error report from " + dnName + ": " + msg); + return; + } + + if (errorCode == DatanodeProtocol.DISK_ERROR) { + LOG.warn("Disk error on " + dnName + ": " + msg); + } else if (errorCode == DatanodeProtocol.FATAL_DISK_ERROR) { + LOG.warn("Fatal disk error on " + dnName + ": " + msg); + blockManager.getDatanodeManager().removeDatanode(registration); + } else { + LOG.info("Error report from " + dnName + ": " + msg); + } + } + + @Override + public NamespaceInfo versionRequest() throws IOException { + ns.readLock(); + try { + return unprotectedGetNamespaceInfo(); + } finally { + ns.readUnlock(); + } + } + + private NamespaceInfo unprotectedGetNamespaceInfo() { + return new NamespaceInfo(1, "random", "random", 2, + NodeType.STORAGE_CONTAINER_SERVICE); + } + + @Override + public void reportBadBlocks(LocatedBlock[] blocks) throws IOException { + // It doesn't make sense to have LocatedBlock in this API. + ns.writeLock(); + try { + for (int i = 0; i < blocks.length; i++) { + ExtendedBlock blk = blocks[i].getBlock(); + DatanodeInfo[] nodes = blocks[i].getLocations(); + String[] storageIDs = blocks[i].getStorageIDs(); + for (int j = 0; j < nodes.length; j++) { + blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], + storageIDs == null ? null: storageIDs[j], + "client machine reported it"); + } + } + } finally { + ns.writeUnlock(); + } + } + + /** + * Start client and service RPC servers. + */ + void start() { + clientRpcServer.start(); + if (serviceRpcServer != null) { + serviceRpcServer.start(); + } + } + + /** + * Wait until the RPC servers have shutdown. + */ + void join() throws InterruptedException { + clientRpcServer.join(); + if (serviceRpcServer != null) { + serviceRpcServer.join(); + } + } + + @Override + public void commitBlockSynchronization(ExtendedBlock block, + long newgenerationstamp, long newlength, boolean closeFile, + boolean deleteblock, DatanodeID[] newtargets, String[] newtargetstorages) + throws IOException { + // Not needed for the purpose of object store + throw new UnsupportedOperationException(); + } + + public static void main(String[] argv) throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(); + StorageContainerManager scm = new StorageContainerManager(conf); + scm.start(); + try { + scm.join(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +}
