Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a070ee4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a070ee4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a070ee4 Branch: refs/heads/HDFS-7240 Commit: 8a070ee489127fdb9d7048b7dbcb7eae74ba539d Parents: c785187 eee0d45 Author: Anu Engineer <[email protected]> Authored: Tue Aug 25 15:20:49 2015 -0700 Committer: Anu Engineer <[email protected]> Committed: Tue Aug 25 15:20:49 2015 -0700 ---------------------------------------------------------------------- BUILDING.txt | 38 +- dev-support/docker/Dockerfile | 8 + dev-support/test-patch.sh | 6 + .../server/AuthenticationFilter.java | 63 +- .../server/AuthenticationToken.java | 12 + .../security/authentication/util/AuthToken.java | 35 +- .../server/TestAuthenticationFilter.java | 163 +++- hadoop-common-project/hadoop-common/CHANGES.txt | 99 ++- hadoop-common-project/hadoop-common/pom.xml | 33 + .../hadoop-common/src/main/bin/hadoop | 8 +- .../hadoop-common/src/main/bin/hadoop-config.sh | 73 +- .../src/main/bin/hadoop-functions.sh | 369 +++++++-- .../hadoop-common/src/main/bin/slaves.sh | 3 +- .../src/main/conf/log4j.properties | 13 + .../org/apache/hadoop/conf/Configuration.java | 9 +- .../org/apache/hadoop/fs/BlockLocation.java | 55 +- .../fs/CommonConfigurationKeysPublic.java | 5 + .../java/org/apache/hadoop/fs/CreateFlag.java | 2 +- .../apache/hadoop/fs/DelegateToFileSystem.java | 19 +- .../main/java/org/apache/hadoop/fs/FsShell.java | 19 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 36 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 11 +- .../hadoop/fs/permission/UmaskParser.java | 2 +- .../org/apache/hadoop/io/ReadaheadPool.java | 4 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 83 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +- .../main/java/org/apache/hadoop/ipc/Server.java | 60 ++ .../apache/hadoop/ipc/WritableRpcEngine.java | 3 + .../apache/hadoop/ipc/metrics/RpcMetrics.java | 54 +- .../metrics2/impl/MetricsSinkAdapter.java | 6 +- .../apache/hadoop/metrics2/lib/MutableStat.java | 7 +- .../org/apache/hadoop/metrics2/util/MBeans.java | 37 +- .../org/apache/hadoop/net/NetworkTopology.java | 38 +- .../apache/hadoop/security/SaslPlainServer.java | 2 +- .../org/apache/hadoop/util/HostsFileReader.java | 7 +- .../org/apache/hadoop/util/JvmPauseMonitor.java | 12 +- .../java/org/apache/hadoop/util/RunJar.java | 3 +- .../main/java/org/apache/hadoop/util/Shell.java | 11 +- .../org/apache/hadoop/util/StringUtils.java | 29 +- .../org/apache/hadoop/io/nativeio/NativeIO.c | 149 ++-- .../src/main/resources/core-default.xml | 27 + .../src/site/markdown/HttpAuthentication.md | 8 +- .../hadoop-common/src/site/markdown/Metrics.md | 4 +- .../src/site/markdown/SingleCluster.md.vm | 2 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 11 +- ...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 2 +- .../fs/FileContextMainOperationsBaseTest.java | 4 +- .../org/apache/hadoop/fs/TestBlockLocation.java | 23 +- .../java/org/apache/hadoop/fs/TestFsShell.java | 28 + .../fs/contract/AbstractContractAppendTest.java | 4 +- .../AbstractContractGetFileStatusTest.java | 61 ++ .../contract/AbstractContractSetTimesTest.java | 61 ++ .../hadoop/fs/contract/ContractOptions.java | 12 + .../TestLocalFSContractGetFileStatus.java | 33 + .../localfs/TestLocalFSContractSetTimes.java | 33 + .../TestRawlocalContractGetFileStatus.java | 33 + .../rawlocal/TestRawlocalContractSetTimes.java | 33 + .../hadoop/io/TestSequenceFileAppend.java | 6 +- .../apache/hadoop/io/nativeio/TestNativeIO.java | 90 ++- .../org/apache/hadoop/ipc/TestProtoBufRpc.java | 77 +- .../metrics2/impl/TestMetricsSystemImpl.java | 60 ++ .../apache/hadoop/test/GenericTestUtils.java | 9 +- .../org/apache/hadoop/test/MetricsAsserts.java | 2 +- .../apache/hadoop/tracing/SetSpanReceiver.java | 2 +- .../java/org/apache/hadoop/util/TestShell.java | 39 + .../hadoop-common/src/test/proto/test.proto | 7 + .../src/test/proto/test_rpc_service.proto | 1 + .../src/test/resources/contract/localfs.xml | 10 + .../src/test/resources/contract/rawlocal.xml | 10 + .../scripts/hadoop-functions_test_helper.bash | 56 ++ .../src/test/scripts/hadoop_add_classpath.bats | 100 +++ .../src/test/scripts/hadoop_add_colonpath.bats | 96 +++ .../scripts/hadoop_add_common_to_classpath.bats | 71 ++ .../test/scripts/hadoop_add_javalibpath.bats | 98 +++ .../src/test/scripts/hadoop_add_ldlibpath.bats | 97 +++ .../src/test/scripts/hadoop_add_param.bats | 49 ++ .../hadoop_add_to_classpath_userpath.bats | 98 +++ .../src/test/scripts/hadoop_basic_init.bats | 94 +++ .../src/test/scripts/hadoop_bootstrap.bats | 51 ++ .../src/test/scripts/hadoop_confdir.bats | 92 +++ .../test/scripts/hadoop_deprecate_envvar.bats | 32 + .../src/test/scripts/hadoop_finalize.bats | 206 +++++ .../scripts/hadoop_finalize_catalina_opts.bats | 56 ++ .../test/scripts/hadoop_finalize_classpath.bats | 64 ++ .../scripts/hadoop_finalize_hadoop_heap.bats | 87 ++ .../scripts/hadoop_finalize_hadoop_opts.bats | 52 ++ .../test/scripts/hadoop_finalize_libpaths.bats | 30 + .../src/test/scripts/hadoop_java_setup.bats | 47 ++ .../src/test/scripts/hadoop_os_tricks.bats | 34 + .../src/test/scripts/hadoop_rotate_log.bats | 52 ++ .../src/test/scripts/hadoop_shellprofile.bats | 91 +++ .../src/test/scripts/hadoop_slaves.bats | 37 + .../src/test/scripts/hadoop_ssh.bats | 51 ++ .../scripts/hadoop_translate_cygwin_path.bats | 48 ++ .../test/scripts/hadoop_validate_classname.bats | 26 + .../hadoop-common/src/test/scripts/run-bats.sh | 43 + .../hadoop-kms/src/main/sbin/kms.sh | 29 +- .../org/apache/hadoop/hdfs/DFSUtilClient.java | 2 + .../org/apache/hadoop/hdfs/ExtendedBlockId.java | 82 ++ .../org/apache/hadoop/hdfs/ReplicaAccessor.java | 88 +++ .../hadoop/hdfs/ReplicaAccessorBuilder.java | 101 +++ .../hdfs/client/HdfsClientConfigKeys.java | 76 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 785 +++++++++++++++++++ .../hadoop/hdfs/client/impl/package-info.java | 18 + .../org/apache/hadoop/hdfs/net/DomainPeer.java | 132 ++++ .../java/org/apache/hadoop/hdfs/net/Peer.java | 123 +++ .../hadoop/hdfs/protocol/ClientProtocol.java | 14 + .../hadoop/hdfs/protocol/HdfsConstants.java | 10 + .../datatransfer/BlockConstructionStage.java | 62 ++ .../datatransfer/DataTransferProtoUtil.java | 146 ++++ .../datatransfer/DataTransferProtocol.java | 202 +++++ .../hadoop/hdfs/protocol/datatransfer/Op.java | 66 ++ .../hdfs/protocol/datatransfer/Sender.java | 261 ++++++ .../hadoop/hdfs/protocolPB/PBHelperClient.java | 254 ++++++ .../token/block/InvalidBlockTokenException.java | 41 + .../hdfs/server/datanode/CachingStrategy.java | 76 ++ .../hadoop/hdfs/shortcircuit/DfsClientShm.java | 119 +++ .../hdfs/shortcircuit/DfsClientShmManager.java | 522 ++++++++++++ .../hdfs/shortcircuit/ShortCircuitShm.java | 647 +++++++++++++++ .../hadoop/hdfs/util/ByteArrayManager.java | 422 ++++++++++ .../hadoop/hdfs/util/ExactSizeInputStream.java | 125 +++ .../apache/hadoop/hdfs/util/package-info.java | 18 + .../hadoop/hdfs/web/WebHdfsFileSystem.java | 20 + .../hdfs/web/resources/CreateFlagParam.java | 48 ++ .../hdfs/web/resources/CreateParentParam.java | 2 +- .../src/main/proto/ClientDatanodeProtocol.proto | 33 - .../src/main/proto/ClientNamenodeProtocol.proto | 10 + .../src/main/proto/datatransfer.proto | 4 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 +- .../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 29 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 208 ++++- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 + .../hadoop-hdfs/src/CMakeLists.txt | 1 + .../hadoop-hdfs/src/main/bin/hdfs | 10 +- .../hadoop-hdfs/src/main/bin/start-balancer.sh | 12 +- .../hadoop-hdfs/src/main/bin/stop-balancer.sh | 10 +- .../apache/hadoop/fs/BlockStorageLocation.java | 51 -- .../java/org/apache/hadoop/fs/HdfsVolumeId.java | 73 -- .../java/org/apache/hadoop/fs/SWebHdfs.java | 51 ++ .../java/org/apache/hadoop/fs/VolumeId.java | 40 - .../main/java/org/apache/hadoop/fs/WebHdfs.java | 51 ++ .../main/java/org/apache/hadoop/fs/package.html | 26 + .../apache/hadoop/hdfs/BlockReaderFactory.java | 60 +- .../hadoop/hdfs/BlockStorageLocationUtil.java | 368 --------- .../org/apache/hadoop/hdfs/ClientContext.java | 5 +- .../java/org/apache/hadoop/hdfs/DFSClient.java | 132 +--- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 195 +++-- .../org/apache/hadoop/hdfs/DFSOutputStream.java | 3 +- .../org/apache/hadoop/hdfs/DataStreamer.java | 6 +- .../hadoop/hdfs/DistributedFileSystem.java | 33 - .../org/apache/hadoop/hdfs/ExtendedBlockId.java | 82 -- .../apache/hadoop/hdfs/ExternalBlockReader.java | 120 +++ .../apache/hadoop/hdfs/HdfsConfiguration.java | 8 +- .../apache/hadoop/hdfs/RemoteBlockReader.java | 4 +- .../apache/hadoop/hdfs/RemoteBlockReader2.java | 4 +- .../org/apache/hadoop/hdfs/XAttrHelper.java | 13 +- .../hadoop/hdfs/client/impl/DfsClientConf.java | 747 ------------------ .../org/apache/hadoop/hdfs/net/DomainPeer.java | 132 ---- .../java/org/apache/hadoop/hdfs/net/Peer.java | 123 --- .../hdfs/protocol/ClientDatanodeProtocol.java | 19 - .../hdfs/protocol/HdfsBlocksMetadata.java | 111 --- .../datatransfer/BlockConstructionStage.java | 62 -- .../datatransfer/DataTransferProtoUtil.java | 148 ---- .../datatransfer/DataTransferProtocol.java | 201 ----- .../hadoop/hdfs/protocol/datatransfer/Op.java | 65 -- .../hdfs/protocol/datatransfer/PipelineAck.java | 2 +- .../hdfs/protocol/datatransfer/Receiver.java | 8 +- .../hdfs/protocol/datatransfer/Sender.java | 261 ------ .../datatransfer/sasl/DataTransferSaslUtil.java | 2 +- ...tDatanodeProtocolServerSideTranslatorPB.java | 43 +- .../ClientDatanodeProtocolTranslatorPB.java | 49 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 23 +- .../ClientNamenodeProtocolTranslatorPB.java | 41 +- .../DatanodeProtocolClientSideTranslatorPB.java | 4 +- .../InterDatanodeProtocolTranslatorPB.java | 2 +- .../NamenodeProtocolTranslatorPB.java | 2 +- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 228 +----- .../token/block/InvalidBlockTokenException.java | 41 - .../hadoop/hdfs/server/balancer/Balancer.java | 182 +++-- .../hadoop/hdfs/server/balancer/Dispatcher.java | 186 +++-- .../hdfs/server/balancer/MovedBlocks.java | 5 + .../server/blockmanagement/BlockCollection.java | 9 +- .../hdfs/server/blockmanagement/BlockInfo.java | 164 ++-- .../blockmanagement/BlockInfoContiguous.java | 82 +- .../BlockInfoUnderConstruction.java | 405 ---------- .../BlockInfoUnderConstructionContiguous.java | 110 --- .../server/blockmanagement/BlockManager.java | 780 ++++++++---------- .../BlockPlacementPolicyDefault.java | 177 +++-- .../BlockPlacementPolicyWithNodeGroup.java | 35 +- .../BlockStoragePolicySuite.java | 11 +- .../BlockUnderConstructionFeature.java | 254 ++++++ .../ContiguousBlockStorageOp.java | 106 --- .../blockmanagement/CorruptReplicasMap.java | 4 +- .../blockmanagement/DatanodeDescriptor.java | 75 +- .../server/blockmanagement/DatanodeManager.java | 19 +- .../blockmanagement/DatanodeStorageInfo.java | 15 +- .../blockmanagement/DecommissionManager.java | 15 +- .../blockmanagement/InvalidateBlocks.java | 2 +- .../ReplicaUnderConstruction.java | 119 +++ .../blockmanagement/UnderReplicatedBlocks.java | 33 +- .../hdfs/server/common/HdfsServerConstants.java | 9 - .../hdfs/server/datanode/BlockReceiver.java | 14 +- .../hdfs/server/datanode/BlockSender.java | 11 +- .../hdfs/server/datanode/CachingStrategy.java | 76 -- .../hadoop/hdfs/server/datanode/DNConf.java | 30 +- .../hadoop/hdfs/server/datanode/DataNode.java | 49 +- .../hdfs/server/datanode/DataXceiver.java | 77 +- .../server/datanode/SecureDataNodeStarter.java | 4 +- .../hdfs/server/datanode/VolumeScanner.java | 12 +- .../server/datanode/fsdataset/FsDatasetSpi.java | 18 +- .../datanode/fsdataset/impl/BlockPoolSlice.java | 2 +- .../datanode/fsdataset/impl/FsDatasetImpl.java | 45 -- .../datanode/web/webhdfs/ParameterParser.java | 14 + .../datanode/web/webhdfs/WebHdfsHandler.java | 23 +- .../hdfs/server/namenode/Checkpointer.java | 4 +- .../hdfs/server/namenode/FSDirAppendOp.java | 4 +- .../hdfs/server/namenode/FSDirAttrOp.java | 73 +- .../hdfs/server/namenode/FSDirConcatOp.java | 9 +- .../hdfs/server/namenode/FSDirDeleteOp.java | 5 +- .../hdfs/server/namenode/FSDirRenameOp.java | 7 +- .../hdfs/server/namenode/FSDirSnapshotOp.java | 2 + .../server/namenode/FSDirStatAndListingOp.java | 9 +- .../hdfs/server/namenode/FSDirTruncateOp.java | 40 +- .../hdfs/server/namenode/FSDirWriteFileOp.java | 30 +- .../hdfs/server/namenode/FSDirXAttrOp.java | 29 +- .../hdfs/server/namenode/FSDirectory.java | 77 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 11 + .../hdfs/server/namenode/FSEditLogLoader.java | 30 +- .../hadoop/hdfs/server/namenode/FSImage.java | 6 +- .../hdfs/server/namenode/FSImageFormat.java | 6 +- .../server/namenode/FSImageFormatPBINode.java | 18 +- .../server/namenode/FSImageSerialization.java | 10 +- .../hdfs/server/namenode/FSNamesystem.java | 161 ++-- .../hdfs/server/namenode/FSNamesystemLock.java | 11 + .../namenode/FileUnderConstructionFeature.java | 10 +- .../hadoop/hdfs/server/namenode/INode.java | 52 +- .../hdfs/server/namenode/INodeDirectory.java | 11 +- .../hadoop/hdfs/server/namenode/INodeFile.java | 60 +- .../hdfs/server/namenode/LeaseManager.java | 17 +- .../hadoop/hdfs/server/namenode/NameNode.java | 176 ++++- .../hdfs/server/namenode/NameNodeRpcServer.java | 6 + .../hdfs/server/namenode/NamenodeFsck.java | 25 +- .../hadoop/hdfs/server/namenode/Namesystem.java | 4 +- .../server/namenode/SerialNumberManager.java | 44 -- .../hdfs/server/namenode/SerialNumberMap.java | 79 ++ .../hdfs/server/namenode/XAttrFeature.java | 78 +- .../hdfs/server/namenode/XAttrFormat.java | 155 ++++ .../server/namenode/XAttrPermissionFilter.java | 6 +- .../hdfs/server/namenode/XAttrStorage.java | 62 +- .../hdfs/server/namenode/ha/StandbyState.java | 3 +- .../namenode/metrics/FSNamesystemMBean.java | 19 + .../snapshot/FSImageFormatPBSnapshot.java | 12 +- .../server/namenode/snapshot/FileDiffList.java | 3 +- .../snapshot/FileWithSnapshotFeature.java | 44 +- .../web/resources/NamenodeWebHdfsMethods.java | 20 +- .../hdfs/server/protocol/NamespaceInfo.java | 17 +- .../hadoop/hdfs/shortcircuit/DfsClientShm.java | 119 --- .../hdfs/shortcircuit/DfsClientShmManager.java | 514 ------------ .../hdfs/shortcircuit/DomainSocketFactory.java | 4 +- .../hdfs/shortcircuit/ShortCircuitCache.java | 4 +- .../hdfs/shortcircuit/ShortCircuitShm.java | 646 --------------- .../offlineImageViewer/FSImageHandler.java | 3 + .../tools/offlineImageViewer/FSImageLoader.java | 90 +++ .../hadoop/hdfs/util/ByteArrayManager.java | 418 ---------- .../hadoop/hdfs/util/ExactSizeInputStream.java | 125 --- .../hadoop/hdfs/util/LightWeightHashSet.java | 21 +- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 6 +- .../StorageContainerNameService.java | 3 +- .../src/main/resources/hdfs-default.xml | 40 +- .../src/main/webapps/hdfs/dfshealth.html | 62 +- .../src/main/webapps/hdfs/dfshealth.js | 63 +- .../webapps/static/dataTables.bootstrap.css | 371 +++++++++ .../main/webapps/static/dataTables.bootstrap.js | 205 +++++ .../src/main/webapps/static/hadoop.css | 51 ++ .../webapps/static/jquery.dataTables.min.js | 160 ++++ .../src/main/webapps/static/moment.min.js | 7 + .../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 5 +- .../src/site/markdown/HdfsImageViewer.md | 1 + .../src/site/markdown/HdfsMultihoming.md | 2 + .../hadoop-hdfs/src/site/markdown/WebHDFS.md | 23 +- .../org/apache/hadoop/hdfs/TestFiPipelines.java | 9 +- .../datanode/TestFiDataTransferProtocol.java | 3 +- .../datanode/TestFiDataTransferProtocol2.java | 5 +- .../hadoop/fs/TestEnhancedByteBufferAccess.java | 10 +- .../TestSWebHdfsFileContextMainOperations.java | 110 +++ .../java/org/apache/hadoop/fs/TestUnbuffer.java | 7 +- .../java/org/apache/hadoop/fs/TestVolumeId.java | 146 ---- .../TestWebHdfsFileContextMainOperations.java | 157 ++++ .../contract/hdfs/TestHDFSContractAppend.java | 15 - .../hdfs/TestHDFSContractGetFileStatus.java | 46 ++ .../contract/hdfs/TestHDFSContractSetTimes.java | 45 ++ .../fs/viewfs/TestViewFsDefaultValue.java | 8 +- .../apache/hadoop/hdfs/BlockReaderTestUtil.java | 6 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 11 +- .../org/apache/hadoop/hdfs/FileAppendTest4.java | 5 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 30 +- .../hadoop/hdfs/TestAppendSnapshotTruncate.java | 13 +- .../hadoop/hdfs/TestBlockReaderFactory.java | 4 +- .../hadoop/hdfs/TestBlockReaderLocal.java | 4 +- .../hadoop/hdfs/TestBlockReaderLocalLegacy.java | 6 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 141 +++- .../TestClientProtocolForPipelineRecovery.java | 4 +- .../org/apache/hadoop/hdfs/TestConnCache.java | 5 +- .../hadoop/hdfs/TestDFSClientRetries.java | 8 +- .../apache/hadoop/hdfs/TestDFSInputStream.java | 2 +- .../hadoop/hdfs/TestDataTransferKeepalive.java | 8 +- .../hadoop/hdfs/TestDataTransferProtocol.java | 6 +- .../apache/hadoop/hdfs/TestDatanodeDeath.java | 5 +- .../apache/hadoop/hdfs/TestDecommission.java | 98 +++ .../hadoop/hdfs/TestDisableConnCache.java | 3 +- .../hadoop/hdfs/TestDistributedFileSystem.java | 263 ++----- .../hadoop/hdfs/TestExternalBlockReader.java | 298 +++++++ .../org/apache/hadoop/hdfs/TestFileAppend.java | 4 +- .../org/apache/hadoop/hdfs/TestFileAppend2.java | 6 +- .../org/apache/hadoop/hdfs/TestFileAppend4.java | 5 +- .../apache/hadoop/hdfs/TestFileCreation.java | 165 ++-- .../java/org/apache/hadoop/hdfs/TestHFlush.java | 3 +- .../apache/hadoop/hdfs/TestParallelRead.java | 2 +- .../TestParallelShortCircuitLegacyRead.java | 4 +- .../TestParallelShortCircuitReadUnCached.java | 6 +- .../hadoop/hdfs/TestParallelUnixDomainRead.java | 2 +- .../org/apache/hadoop/hdfs/TestPipelines.java | 9 +- .../java/org/apache/hadoop/hdfs/TestPread.java | 6 +- .../java/org/apache/hadoop/hdfs/TestRead.java | 5 +- .../hadoop/hdfs/TestRemoteBlockReader.java | 4 +- .../hdfs/TestReplaceDatanodeOnFailure.java | 4 +- .../org/apache/hadoop/hdfs/TestReplication.java | 26 +- .../hdfs/protocol/TestBlockListAsLongs.java | 9 +- .../hadoop/hdfs/protocolPB/TestPBHelper.java | 20 +- .../hdfs/server/balancer/TestBalancer.java | 188 ++++- .../server/blockmanagement/TestBlockInfo.java | 2 +- .../TestBlockInfoUnderConstruction.java | 82 -- .../blockmanagement/TestBlockManager.java | 16 +- .../TestBlockReportRateLimiting.java | 2 - .../blockmanagement/TestBlockTokenWithDFS.java | 6 +- .../TestBlockUnderConstructionFeature.java | 80 ++ .../blockmanagement/TestHeartbeatHandling.java | 22 +- .../server/blockmanagement/TestNodeCount.java | 2 +- .../TestOverReplicatedBlocks.java | 4 +- .../blockmanagement/TestPendingReplication.java | 1 - .../blockmanagement/TestReplicationPolicy.java | 98 ++- .../TestUnderReplicatedBlocks.java | 48 -- .../server/datanode/SimulatedFSDataset.java | 8 - .../server/datanode/TestBlockReplacement.java | 7 +- .../server/datanode/TestCachingStrategy.java | 12 +- .../datanode/TestDataNodeVolumeFailure.java | 6 +- .../TestDataXceiverLazyPersistHint.java | 180 +++++ .../extdataset/ExternalDatasetImpl.java | 6 - .../fsdataset/impl/LazyPersistTestCase.java | 5 +- .../fsdataset/impl/TestDatanodeRestart.java | 7 +- .../hdfs/server/mover/TestStorageMover.java | 8 +- .../namenode/TestBlockUnderConstruction.java | 6 +- .../TestCommitBlockSynchronization.java | 12 +- .../TestDefaultBlockPlacementPolicy.java | 49 +- .../hdfs/server/namenode/TestFSNamesystem.java | 28 + .../server/namenode/TestFSNamesystemMBean.java | 34 +- .../hdfs/server/namenode/TestFileTruncate.java | 5 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 2 + .../hdfs/server/namenode/TestHDFSConcat.java | 24 +- .../hdfs/server/namenode/TestINodeFile.java | 7 +- .../hdfs/server/namenode/TestLeaseManager.java | 65 +- .../server/namenode/TestNameNodeMXBean.java | 23 +- .../namenode/TestNameNodeMetricsLogger.java | 193 +++++ .../server/namenode/TestQuotaByStorageType.java | 14 +- .../hdfs/server/namenode/TestStartup.java | 27 +- .../hdfs/server/namenode/TestXAttrFeature.java | 107 +++ .../namenode/ha/TestRetryCacheWithHA.java | 11 +- .../server/namenode/ha/TestStandbyIsHot.java | 2 + .../namenode/snapshot/SnapshotTestHelper.java | 9 +- .../snapshot/TestFileWithSnapshotFeature.java | 7 +- .../namenode/snapshot/TestSnapshotDeletion.java | 16 +- .../snapshot/TestSnapshotReplication.java | 31 +- .../shortcircuit/TestShortCircuitCache.java | 13 +- .../shortcircuit/TestShortCircuitLocalRead.java | 6 +- ...TestOfflineImageViewerForContentSummary.java | 248 ++++++ .../hdfs/util/TestLightWeightHashSet.java | 29 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 25 + .../apache/hadoop/net/TestNetworkTopology.java | 1 + .../apache/hadoop/security/TestPermission.java | 26 + .../src/test/resources/contract/hdfs.xml | 10 + .../src/test/resources/log4j.properties | 13 + .../src/test/resources/testHDFSConf.xml | 2 +- hadoop-mapreduce-project/CHANGES.txt | 34 + hadoop-mapreduce-project/bin/mapred | 2 +- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 2 +- .../v2/app/job/event/JobStartEvent.java | 2 +- .../mapreduce/v2/app/job/impl/JobImpl.java | 40 +- .../mapreduce/v2/app/rm/RMCommunicator.java | 51 +- .../app/rm/RMContainerAllocationException.java | 31 + .../v2/app/rm/RMContainerAllocator.java | 4 +- .../v2/api/records/TestTaskAttemptReport.java | 131 ++++ .../v2/api/records/TestTaskReport.java | 139 ++++ .../mapreduce/v2/app/TestMRAppMaster.java | 88 ++- .../mapreduce/v2/app/job/impl/TestJobImpl.java | 130 ++- .../mapreduce/v2/app/rm/TestRMCommunicator.java | 99 +++ .../v2/app/rm/TestRMContainerAllocator.java | 48 +- .../v2/api/records/TaskAttemptReport.java | 3 + .../mapreduce/v2/api/records/TaskReport.java | 5 +- .../impl/pb/TaskAttemptReportPBImpl.java | 38 +- .../api/records/impl/pb/TaskReportPBImpl.java | 38 +- .../java/org/apache/hadoop/mapred/TaskLog.java | 4 +- .../mapreduce/jobhistory/EventWriter.java | 19 +- .../mapreduce/lib/output/MultipleOutputs.java | 14 +- .../src/main/resources/mapred-default.xml | 20 + .../src/site/markdown/EncryptedShuffle.md | 8 +- .../hadoop/mapreduce/v2/hs/CompletedTask.java | 2 +- .../mapreduce/v2/hs/CompletedTaskAttempt.java | 2 +- .../mapreduce/v2/hs/JobHistoryServer.java | 14 +- .../hadoop/mapred/ResourceMgrDelegate.java | 7 + .../hadoop/mapred/TestClientRedirect.java | 9 + .../hadoop/mapred/FadvisedChunkedFile.java | 5 +- .../hadoop/mapred/FadvisedFileRegion.java | 6 +- hadoop-project/pom.xml | 4 +- hadoop-tools/hadoop-aws/pom.xml | 2 +- .../org/apache/hadoop/fs/s3a/Constants.java | 9 +- .../hadoop/fs/s3a/S3AFastOutputStream.java | 2 +- .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 26 +- .../apache/hadoop/fs/s3a/S3AOutputStream.java | 15 +- .../src/site/markdown/tools/hadoop-aws/index.md | 6 + .../fs/azure/TestWasbUriAndConfiguration.java | 30 +- .../src/test/resources/azure-test.xml | 4 + .../org/apache/hadoop/tools/CopyListing.java | 15 +- .../java/org/apache/hadoop/tools/DiffInfo.java | 32 +- .../java/org/apache/hadoop/tools/DistCp.java | 27 +- .../org/apache/hadoop/tools/DistCpOptions.java | 4 +- .../org/apache/hadoop/tools/DistCpSync.java | 308 +++++++- .../apache/hadoop/tools/SimpleCopyListing.java | 151 +++- .../org/apache/hadoop/tools/TestDistCpSync.java | 345 +++++++- .../apache/hadoop/tools/TestOptionsParser.java | 22 +- .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 3 + .../yarn/sls/scheduler/RMNodeWrapper.java | 5 + hadoop-yarn-project/CHANGES.txt | 135 +++- .../hadoop-yarn/bin/start-yarn.sh | 7 +- .../hadoop-yarn/bin/stop-yarn.sh | 7 +- hadoop-yarn-project/hadoop-yarn/bin/yarn | 7 + .../yarn/api/ApplicationClientProtocol.java | 18 + .../UpdateApplicationPriorityRequest.java | 80 ++ .../UpdateApplicationPriorityResponse.java | 47 ++ .../yarn/api/records/ApplicationReport.java | 31 +- .../yarn/api/records/LogAggregationContext.java | 95 +++ .../yarn/api/records/QueueStatistics.java | 36 + .../hadoop/yarn/conf/YarnConfiguration.java | 61 +- .../api/ContainerLogAggregationPolicy.java | 54 ++ .../yarn/server/api/ContainerLogContext.java | 71 ++ .../main/proto/applicationclient_protocol.proto | 1 + .../src/main/proto/yarn_protos.proto | 7 + .../src/main/proto/yarn_service_protos.proto | 8 + .../distributedshell/ApplicationMaster.java | 31 +- .../distributedshell/TestDSAppMaster.java | 11 +- .../hadoop/yarn/client/api/YarnClient.java | 17 + .../yarn/client/api/impl/YarnClientImpl.java | 11 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 35 +- .../apache/hadoop/yarn/client/cli/TopCLI.java | 24 +- .../yarn/client/TestResourceTrackerOnHA.java | 2 +- .../yarn/client/api/impl/TestYarnClient.java | 10 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 34 +- .../ApplicationClientProtocolPBClientImpl.java | 20 + .../ApplicationClientProtocolPBServiceImpl.java | 22 + .../UpdateApplicationPriorityRequestPBImpl.java | 171 ++++ ...UpdateApplicationPriorityResponsePBImpl.java | 69 ++ .../impl/pb/ApplicationReportPBImpl.java | 49 ++ .../impl/pb/LogAggregationContextPBImpl.java | 40 + .../records/impl/pb/QueueStatisticsPBImpl.java | 36 + .../hadoop/yarn/event/AsyncDispatcher.java | 9 +- .../ContainerLogsRetentionPolicy.java | 29 - .../nodelabels/CommonNodeLabelsManager.java | 22 +- .../nodelabels/FileSystemNodeLabelsStore.java | 8 +- .../hadoop/yarn/util/resource/Resources.java | 14 +- .../src/main/resources/yarn-default.xml | 99 ++- .../hadoop/yarn/api/TestApplicatonReport.java | 4 +- .../hadoop/yarn/event/TestAsyncDispatcher.java | 25 +- .../nodelabels/TestCommonNodeLabelsManager.java | 17 +- ...pplicationHistoryManagerOnTimelineStore.java | 28 +- .../ApplicationHistoryServer.java | 14 +- ...pplicationHistoryManagerOnTimelineStore.java | 4 + .../yarn/server/api/records/NodeStatus.java | 18 +- .../api/records/impl/pb/NodeStatusPBImpl.java | 22 + .../metrics/ApplicationMetricsConstants.java | 6 + .../hadoop/yarn/server/utils/BuilderUtils.java | 4 +- .../hadoop/yarn/server/webapp/AppBlock.java | 24 +- .../hadoop/yarn/server/webapp/AppsBlock.java | 12 +- .../hadoop/yarn/server/webapp/WebPageUtils.java | 4 +- .../hadoop/yarn/server/webapp/dao/AppInfo.java | 15 + .../main/proto/yarn_server_common_protos.proto | 1 + .../hadoop/yarn/server/nodemanager/Context.java | 2 + .../yarn/server/nodemanager/NodeManager.java | 73 +- .../server/nodemanager/NodeResourceMonitor.java | 10 +- .../nodemanager/NodeResourceMonitorImpl.java | 140 ++++ .../nodemanager/NodeStatusUpdaterImpl.java | 280 +++++-- .../application/ApplicationImpl.java | 5 +- .../AMOnlyLogAggregationPolicy.java | 31 + ...AMOrFailedContainerLogAggregationPolicy.java | 35 + .../AbstractContainerLogAggregationPolicy.java | 31 + .../AllContainerLogAggregationPolicy.java | 30 + .../logaggregation/AppLogAggregator.java | 5 +- .../logaggregation/AppLogAggregatorImpl.java | 131 ++-- .../FailedContainerLogAggregationPolicy.java | 33 + ...edOrKilledContainerLogAggregationPolicy.java | 30 + .../logaggregation/LogAggregationService.java | 19 +- .../NoneContainerLogAggregationPolicy.java | 30 + .../SampleContainerLogAggregationPolicy.java | 124 +++ .../event/LogHandlerAppStartedEvent.java | 15 +- .../monitor/ContainersMonitorImpl.java | 18 +- .../nodemanager/metrics/NodeManagerMetrics.java | 13 +- .../nodelabels/AbstractNodeLabelsProvider.java | 146 ++++ .../ConfigurationNodeLabelsProvider.java | 81 ++ .../server/nodemanager/webapp/NodePage.java | 8 +- .../server/nodemanager/webapp/dao/NodeInfo.java | 6 + .../impl/container-executor.c | 63 +- .../server/nodemanager/TestNodeManager.java | 50 +- .../nodemanager/TestNodeResourceMonitor.java | 35 + .../nodemanager/TestNodeStatusUpdater.java | 34 + .../TestNodeStatusUpdaterForLabels.java | 76 +- .../containermanager/TestAuxServices.java | 1 + .../launcher/TestContainerLaunch.java | 2 +- .../TestLogAggregationService.java | 677 ++++++++++++++-- .../TestNonAggregatingLogHandler.java | 12 +- .../monitor/TestContainersMonitor.java | 2 +- .../TestConfigurationNodeLabelsProvider.java | 146 ++++ .../nodemanager/webapp/TestNMWebServices.java | 2 +- .../ApplicationMasterService.java | 3 +- .../server/resourcemanager/ClientRMService.java | 84 +- .../resourcemanager/NodesListManager.java | 28 +- .../resourcemanager/RMActiveServiceContext.java | 30 - .../server/resourcemanager/RMAuditLogger.java | 2 + .../server/resourcemanager/RMContextImpl.java | 23 +- .../server/resourcemanager/ResourceManager.java | 43 +- .../resourcemanager/ResourceTrackerService.java | 2 + .../metrics/ApplicationCreatedEvent.java | 17 +- .../metrics/SystemMetricsPublisher.java | 9 +- .../nodelabels/RMNodeLabelsManager.java | 14 +- .../recovery/FileSystemRMStateStore.java | 209 +++-- .../recovery/LeveldbRMStateStore.java | 114 ++- .../recovery/MemoryRMStateStore.java | 57 ++ .../recovery/NullRMStateStore.java | 23 + .../resourcemanager/recovery/RMStateStore.java | 166 +++- .../recovery/RMStateStoreEventType.java | 5 +- .../RMStateStoreStoreReservationEvent.java | 56 ++ .../recovery/RMStateUpdateAppEvent.java | 13 + .../recovery/ZKRMStateStore.java | 124 ++- .../resourcemanager/reservation/Plan.java | 3 +- .../resourcemanager/reservation/PlanView.java | 11 +- .../ReservationSchedulerConfiguration.java | 14 +- .../reservation/ReservationSystem.java | 15 +- .../reservation/ReservationSystemUtil.java | 102 ++- .../server/resourcemanager/rmapp/RMAppImpl.java | 12 +- .../rmapp/attempt/RMAppAttemptImpl.java | 22 +- .../rmcontainer/RMContainerImpl.java | 70 +- .../server/resourcemanager/rmnode/RMNode.java | 7 +- .../resourcemanager/rmnode/RMNodeImpl.java | 23 +- .../scheduler/AbstractYarnScheduler.java | 17 +- .../scheduler/SchedulerApplicationAttempt.java | 2 +- .../scheduler/YarnScheduler.java | 22 + .../scheduler/capacity/AbstractCSQueue.java | 11 +- .../scheduler/capacity/CSAssignment.java | 11 +- .../scheduler/capacity/CapacityScheduler.java | 115 ++- .../capacity/CapacitySchedulerContext.java | 2 - .../scheduler/capacity/LeafQueue.java | 90 ++- .../scheduler/capacity/ParentQueue.java | 18 +- .../capacity/allocator/AllocationState.java | 28 + .../capacity/allocator/ContainerAllocation.java | 89 +++ .../capacity/allocator/ContainerAllocator.java | 150 ++++ .../allocator/RegularContainerAllocator.java | 698 +++++++++++++++++ .../scheduler/common/fica/FiCaSchedulerApp.java | 639 +-------------- .../scheduler/fair/FSAppAttempt.java | 4 +- .../AbstractComparatorOrderingPolicy.java | 6 + .../ClientToAMTokenSecretManagerInRM.java | 7 + .../resourcemanager/webapp/RMAppsBlock.java | 7 +- .../resourcemanager/webapp/dao/AppInfo.java | 19 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 18 + .../webapp/dao/FairSchedulerQueueInfo.java | 24 +- .../yarn_server_resourcemanager_recovery.proto | 18 +- .../yarn/server/resourcemanager/MockAM.java | 14 +- .../yarn/server/resourcemanager/MockNodes.java | 4 + .../server/resourcemanager/TestAppManager.java | 5 +- .../resourcemanager/TestClientRMService.java | 64 ++ .../resourcemanager/TestRMNodeTransitions.java | 62 +- .../resourcemanager/TestResourceManager.java | 1 + .../TestWorkPreservingRMRestart.java | 2 +- .../TestRMAppLogAggregationStatus.java | 4 +- .../metrics/TestSystemMetricsPublisher.java | 16 + ...pacityPreemptionPolicyForNodePartitions.java | 10 +- .../nodelabels/TestRMNodeLabelsManager.java | 48 +- .../recovery/RMStateStoreTestBase.java | 186 +++++ .../recovery/TestFSRMStateStore.java | 1 + .../recovery/TestLeveldbRMStateStore.java | 6 + .../recovery/TestZKRMStateStore.java | 1 + .../reservation/ReservationSystemTestUtil.java | 193 ++--- .../TestCapacityReservationSystem.java | 94 --- .../TestCapacitySchedulerPlanFollower.java | 11 +- .../reservation/TestFairReservationSystem.java | 127 --- .../TestFairSchedulerPlanFollower.java | 71 +- .../TestInMemoryReservationAllocation.java | 36 +- .../reservation/TestReservationSystem.java | 213 +++++ .../resourcetracker/TestNMReconnect.java | 39 + .../rmapp/TestNodesListManager.java | 162 ++++ .../rmapp/TestRMAppTransitions.java | 6 +- .../attempt/TestRMAppAttemptTransitions.java | 40 +- .../rmcontainer/TestRMContainerImpl.java | 79 +- .../capacity/TestApplicationLimits.java | 14 +- .../capacity/TestApplicationPriority.java | 329 ++++++-- .../capacity/TestCapacityScheduler.java | 83 +- .../scheduler/capacity/TestChildQueueOrder.java | 3 +- .../capacity/TestContainerAllocation.java | 12 +- .../scheduler/capacity/TestLeafQueue.java | 356 +++++---- .../scheduler/capacity/TestParentQueue.java | 2 - .../scheduler/capacity/TestReservations.java | 8 +- .../scheduler/capacity/TestUtils.java | 9 +- .../scheduler/fifo/TestFifoScheduler.java | 19 +- .../webapp/TestRMWebServicesApps.java | 36 +- .../webapp/TestRMWebServicesCapacitySched.java | 4 +- .../yarn/server/webproxy/WebAppProxyServer.java | 32 +- .../src/site/markdown/ResourceManagerRest.md | 20 +- .../src/site/markdown/TimelineServer.md | 32 +- 614 files changed, 24687 insertions(+), 10842 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java index 7131c7d,42934c3..786dd1a --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfoContiguous.java @@@ -19,10 -19,7 +19,9 @@@ package org.apache.hadoop.hdfs.server.b import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hdfs.protocol.Block; - import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; +import java.util.LinkedList; + /** * Subclass of {@link BlockInfo}, used for a block with replication scheme. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 144d8fb,7f02612..4dd2205 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@@ -91,14 -99,6 +99,7 @@@ import com.google.common.base.Precondit import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.IOException; - import java.io.PrintWriter; - import java.util.*; - import java.util.concurrent.ThreadLocalRandom; - import java.util.concurrent.atomic.AtomicLong; - - import static org.apache.hadoop.util.ExitUtil.terminate; + /** * Keeps information related to the blocks stored in the Hadoop cluster. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index de30e08,7e3c59b..fd1c0f0 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@@ -17,10 -17,22 +17,8 @@@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; - import com.google.common.annotations.VisibleForTesting; - import com.google.common.collect.ImmutableList; - import org.apache.commons.logging.Log; - import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.StorageType; @@@ -36,9 -49,9 +35,11 @@@ import org.apache.hadoop.hdfs.util.Enum import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.IntrusiveCollection; import org.apache.hadoop.util.Time; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; +import java.util.*; + /** * This class extends the DatanodeInfo class with ephemeral information (eg * health, capacity, what blocks are associated with the Datanode) that is http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java index a325a19,216d6d2..8bd8c51 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java @@@ -17,9 -17,13 +17,8 @@@ */ package org.apache.hadoop.hdfs.server.blockmanagement; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - import com.google.common.annotations.VisibleForTesting; - import org.apache.hadoop.fs.StorageType; - import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java index f393aa0,1cb308f..2468453 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java @@@ -791,15 -791,15 +794,15 @@@ class BlockReceiver implements Closeabl // if (syncBehindWrites) { if (syncBehindWritesInBackground) { - this.datanode.getFSDataset().submitBackgroundSyncFileRangeRequest( + dataset.submitBackgroundSyncFileRangeRequest( block, outFd, lastCacheManagementOffset, offsetInBlock - lastCacheManagementOffset, - NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); + SYNC_FILE_RANGE_WRITE); } else { NativeIO.POSIX.syncFileRangeIfPossible(outFd, - lastCacheManagementOffset, offsetInBlock - - lastCacheManagementOffset, - NativeIO.POSIX.SYNC_FILE_RANGE_WRITE); + lastCacheManagementOffset, + offsetInBlock - lastCacheManagementOffset, + SYNC_FILE_RANGE_WRITE); } } // http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java index 75b0d26,dfaa525..11ad30a --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java @@@ -643,14 -628,9 +642,16 @@@ class DataXceiver extends Receiver impl final boolean isClient = !isDatanode; final boolean isTransfer = stage == BlockConstructionStage.TRANSFER_RBW || stage == BlockConstructionStage.TRANSFER_FINALIZED; + allowLazyPersist = allowLazyPersist && + (dnConf.getAllowNonLocalLazyPersist() || peer.isLocal()); long size = 0; + final FsDatasetSpi<?> dataset = + (FsDatasetSpi<?>) datanode.getDataset(block.getBlockPoolId()); + if (dataset == null) { + throw new IOException( + "Unknown or unitialized blockpool " + block.getBlockPoolId()); + } + // check single target for transfer-RBW/Finalized if (isTransfer && targets.length > 0) { throw new IOException(stage + " does not support multiple targets " @@@ -1208,10 -1158,10 +1205,10 @@@ DataChecksum remoteChecksum = DataTransferProtoUtil.fromProto( checksumInfo.getChecksum()); // open a block receiver and check if the block does not exist - blockReceiver = new BlockReceiver(block, storageType, + blockReceiver = getBlockReceiver(block, storageType, proxyReply, proxySock.getRemoteSocketAddress().toString(), proxySock.getLocalSocketAddress().toString(), - null, 0, 0, 0, "", null, datanode, remoteChecksum, + null, 0, 0, 0, "", null, datanode, dataset, remoteChecksum, CachingStrategy.newDropBehind(), false, false); // receive a block @@@ -1263,6 -1213,39 +1260,40 @@@ datanode.metrics.addReplaceBlockOp(elapsed()); } + + /** + * Separated for testing. + */ + @VisibleForTesting + BlockReceiver getBlockReceiver( + final ExtendedBlock block, final StorageType storageType, + final DataInputStream in, + final String inAddr, final String myAddr, + final BlockConstructionStage stage, + final long newGs, final long minBytesRcvd, final long maxBytesRcvd, + final String clientname, final DatanodeInfo srcDataNode, - final DataNode dn, DataChecksum requestedChecksum, ++ final DataNode dn, final FsDatasetSpi<?> dataset, ++ DataChecksum requestedChecksum, + CachingStrategy cachingStrategy, + final boolean allowLazyPersist, + final boolean pinning) throws IOException { + return new BlockReceiver(block, storageType, in, + inAddr, myAddr, stage, newGs, minBytesRcvd, maxBytesRcvd, - clientname, srcDataNode, dn, requestedChecksum, ++ clientname, srcDataNode, dn, dataset, requestedChecksum, + cachingStrategy, allowLazyPersist, pinning); + } + + /** + * Separated for testing. + * @return + */ + @VisibleForTesting + DataOutputStream getBufferedOutputStream() { + return new DataOutputStream( + new BufferedOutputStream(getOutputStream(), smallBufferSize)); + } + + private long elapsed() { return monotonicNow() - opStartTime; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index 3c39ace,63f0377..9b9e5c4 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@@ -30,14 -30,19 +30,13 @@@ import java.util.ArrayList import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.protocol.Block; -import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; - import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; -import org.apache.hadoop.hdfs.server.datanode.DataNode; -import org.apache.hadoop.hdfs.server.datanode.DataStorage; import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica; import org.apache.hadoop.hdfs.server.datanode.Replica; import org.apache.hadoop.hdfs.server.datanode.ReplicaInPipelineInterface; @@@ -398,21 -560,37 +397,18 @@@ public interface FsDatasetSpi<V extend ) throws IOException; /** - * Get a {@link HdfsBlocksMetadata} corresponding to the list of blocks in - * <code>blocks</code>. - * - * @param bpid pool to query - * @param blockIds List of block ids for which to return metadata - * @return metadata Metadata for the list of blocks - * @throws IOException ++ /** ++ * submit a sync_file_range request to AsyncDiskService + * Enable 'trash' for the given dataset. When trash is enabled, files are + * moved to a separate trash directory instead of being deleted immediately. + * This can be useful for example during rolling upgrades. */ - public HdfsBlocksMetadata getHdfsBlocksMetadata(String bpid, - long[] blockIds) throws IOException; + void enableTrash(String bpid); /** - * submit a sync_file_range request to AsyncDiskService + * Clear trash */ - void clearTrash(String bpid); - - /** - * @return true when trash is enabled - */ - boolean trashEnabled(String bpid); - - /** - * Create a marker file indicating that a rolling upgrade is in progress. - */ - void setRollingUpgradeMarker(String bpid) throws IOException; - - /** - * Delete the rolling upgrade marker file if it exists. - * @param bpid - */ - void clearRollingUpgradeMarker(String bpid) throws IOException; - - /** - * submit a sync_file_range request to AsyncDiskService. - */ - void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block, + public void submitBackgroundSyncFileRangeRequest(final ExtendedBlock block, final FileDescriptor fd, final long offset, final long nbytes, final int flags); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java index 2b7e3ff,0000000..7d1eb66 mode 100644,000000..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/storagecontainer/StorageContainerNameService.java @@@ -1,163 -1,0 +1,162 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.storagecontainer; + +import org.apache.hadoop.hdfs.protocol.Block; +import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo; - import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction; +import org.apache.hadoop.hdfs.server.namenode.CacheManager; +import org.apache.hadoop.hdfs.server.namenode.NameNode; +import org.apache.hadoop.hdfs.server.namenode.Namesystem; +import org.apache.hadoop.ipc.StandbyException; +import org.apache.hadoop.security.AccessControlException; + +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * Namesystem implementation to be used by StorageContainerManager. + */ +public class StorageContainerNameService implements Namesystem { + + private ReentrantReadWriteLock coarseLock = new ReentrantReadWriteLock(); + private String blockPoolId; + private volatile boolean serviceRunning = true; + + public void shutdown() { + serviceRunning = false; + } + + @Override + public boolean isRunning() { + return serviceRunning; + } + + @Override + public void checkSuperuserPrivilege() throws AccessControlException { + // TBD + } + + @Override + public String getBlockPoolId() { + return blockPoolId; + } + + public void setBlockPoolId(String id) { + this.blockPoolId = id; + } + + @Override + public boolean isInStandbyState() { + // HA mode is not supported + return false; + } + + @Override + public boolean isGenStampInFuture(Block block) { + // HA mode is not supported + return false; + } + + @Override + public void adjustSafeModeBlockTotals(int deltaSafe, int deltaTotal) { + // TBD + } + + @Override + public void checkOperation(NameNode.OperationCategory read) + throws StandbyException { + // HA mode is not supported + } + + @Override - public boolean isInSnapshot(BlockInfoUnderConstruction blockUC) { ++ public boolean isInSnapshot(BlockInfo blockInfo) { + // Snapshots not supported + return false; + } + + @Override + public CacheManager getCacheManager() { + // Cache Management is not supported + return null; + } + + @Override + public void readLock() { + coarseLock.readLock().lock(); + } + + @Override + public void readUnlock() { + coarseLock.readLock().unlock(); + } + + @Override + public boolean hasReadLock() { + return coarseLock.getReadHoldCount() > 0 || hasWriteLock(); + } + + @Override + public void writeLock() { + coarseLock.writeLock().lock(); + } + + @Override + public void writeLockInterruptibly() throws InterruptedException { + coarseLock.writeLock().lockInterruptibly(); + } + + @Override + public void writeUnlock() { + coarseLock.writeLock().unlock(); + } + + @Override + public boolean hasWriteLock() { + return coarseLock.isWriteLockedByCurrentThread(); + } + + @Override + public void checkSafeMode() { + // TBD + } + + @Override + public boolean isInSafeMode() { + return false; + } + + @Override + public boolean isInStartupSafeMode() { + return false; + } + + @Override + public boolean isPopulatingReplQueues() { + return false; + } + + @Override + public void incrementSafeBlockCount(int replication) { + // Do nothing + } + + @Override + public void decrementSafeBlockCount(BlockInfo b) { + // Do nothing + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java index b05be39,5d1b31a..4788fcc --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java @@@ -42,12 -42,7 +42,10 @@@ import org.apache.hadoop.hdfs.protocol. import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; - import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; - import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.common.Storage; +import org.apache.hadoop.hdfs.server.datanode.dataset.DatasetSpi; +import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java index 0000000,d8a7188..48f8cef mode 000000,100644..100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataXceiverLazyPersistHint.java @@@ -1,0 -1,178 +1,180 @@@ + /** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.hadoop.hdfs.server.datanode; + + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.fs.StorageType; + import org.apache.hadoop.hdfs.*; + import org.apache.hadoop.hdfs.net.*; + import org.apache.hadoop.hdfs.protocol.*; + import org.apache.hadoop.hdfs.protocol.datatransfer.*; + import org.apache.hadoop.hdfs.server.datanode.metrics.DataNodeMetrics; + import org.apache.hadoop.util.DataChecksum; + import static org.apache.hadoop.hdfs.DFSConfigKeys.*; ++import org.apache.hadoop.hdfs.server.datanode.fsdataset.*; + + import org.junit.Rule; + import org.junit.Test; + import org.junit.rules.Timeout; + import org.mockito.ArgumentCaptor; + + import java.io.DataInputStream; + import java.io.DataOutputStream; + import java.io.IOException; + import java.util.Arrays; + + import static org.hamcrest.MatcherAssert.assertThat; + import static org.hamcrest.core.Is.is; + import static org.mockito.Mockito.*; + + + /** + * Mock-based unit test to verify that the DataXceiver correctly handles the + * LazyPersist hint from clients. + */ + public class TestDataXceiverLazyPersistHint { + @Rule + public Timeout timeout = new Timeout(300000); + + private enum PeerLocality { + LOCAL, + REMOTE + } + + private enum NonLocalLazyPersist { + ALLOWED, + NOT_ALLOWED + } + + /** + * Ensure that the correct hint is passed to the block receiver when + * the client is local. + */ + @Test + public void testWithLocalClient() throws IOException { + ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class); + DataXceiver xceiver = makeStubDataXceiver( + PeerLocality.LOCAL, NonLocalLazyPersist.NOT_ALLOWED, captor); + + for (Boolean lazyPersistSetting : Arrays.asList(true, false)) { + issueWriteBlockCall(xceiver, lazyPersistSetting); + assertThat(captor.getValue(), is(lazyPersistSetting)); + } + } + + /** + * Ensure that hint is always false when the client is remote. + */ + @Test + public void testWithRemoteClient() throws IOException { + ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class); + DataXceiver xceiver = makeStubDataXceiver( + PeerLocality.REMOTE, NonLocalLazyPersist.NOT_ALLOWED, captor); + + for (Boolean lazyPersistSetting : Arrays.asList(true, false)) { + issueWriteBlockCall(xceiver, lazyPersistSetting); + assertThat(captor.getValue(), is(false)); + } + } + + /** + * Ensure that the correct hint is passed to the block receiver when + * the client is remote AND dfs.datanode.allow.non.local.lazy.persist + * is set to true. + */ + @Test + public void testOverrideWithRemoteClient() throws IOException { + ArgumentCaptor<Boolean> captor = ArgumentCaptor.forClass(Boolean.class); + DataXceiver xceiver = makeStubDataXceiver( + PeerLocality.REMOTE, NonLocalLazyPersist.ALLOWED, captor); + + for (Boolean lazyPersistSetting : Arrays.asList(true, false)) { + issueWriteBlockCall(xceiver, lazyPersistSetting); + assertThat(captor.getValue(), is(lazyPersistSetting)); + } + } + + /** + * Issue a write block call with dummy parameters. The only parameter useful + * for this test is the value of lazyPersist. + */ + private void issueWriteBlockCall(DataXceiver xceiver, boolean lazyPersist) + throws IOException { + xceiver.writeBlock( + new ExtendedBlock("Dummy-pool", 0L), + StorageType.RAM_DISK, + null, + "Dummy-Client", + new DatanodeInfo[0], + new StorageType[0], + mock(DatanodeInfo.class), + BlockConstructionStage.PIPELINE_SETUP_CREATE, + 0, 0, 0, 0, + DataChecksum.newDataChecksum(DataChecksum.Type.NULL, 0), + CachingStrategy.newDefaultStrategy(), + lazyPersist, + false, null); + } + + // Helper functions to setup the mock objects. + + private static DataXceiver makeStubDataXceiver( + PeerLocality locality, + NonLocalLazyPersist nonLocalLazyPersist, + final ArgumentCaptor<Boolean> captor) throws IOException { + DataXceiver xceiverSpy = spy(DataXceiver.create( + getMockPeer(locality), + getMockDn(nonLocalLazyPersist), + mock(DataXceiverServer.class))); + + doReturn(mock(BlockReceiver.class)).when(xceiverSpy).getBlockReceiver( + any(ExtendedBlock.class), any(StorageType.class), + any(DataInputStream.class), anyString(), anyString(), + any(BlockConstructionStage.class), anyLong(), anyLong(), anyLong(), + anyString(), any(DatanodeInfo.class), any(DataNode.class), ++ any(FsDatasetSpi.class), + any(DataChecksum.class), any(CachingStrategy.class), + captor.capture(), anyBoolean()); + doReturn(mock(DataOutputStream.class)).when(xceiverSpy) + .getBufferedOutputStream(); + return xceiverSpy; + } + + private static Peer getMockPeer(PeerLocality locality) { + Peer peer = mock(Peer.class); + when(peer.isLocal()).thenReturn(locality == PeerLocality.LOCAL); + when(peer.getRemoteAddressString()).thenReturn("1.1.1.1:1000"); + when(peer.getLocalAddressString()).thenReturn("2.2.2.2:2000"); + return peer; + } + + private static DataNode getMockDn(NonLocalLazyPersist nonLocalLazyPersist) { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean( + DFS_DATANODE_NON_LOCAL_LAZY_PERSIST, + nonLocalLazyPersist == NonLocalLazyPersist.ALLOWED); + DNConf dnConf = new DNConf(conf); + DataNodeMetrics mockMetrics = mock(DataNodeMetrics.class); + DataNode mockDn = mock(DataNode.class); + when(mockDn.getDnConf()).thenReturn(dnConf); + when(mockDn.getConf()).thenReturn(conf); + when(mockDn.getMetrics()).thenReturn(mockMetrics); + return mockDn; + } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a070ee4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java index 29bb04b,e3be5fd..c5a1bd9 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java @@@ -27,9 -27,7 +27,8 @@@ import org.apache.hadoop.hdfs.protocol. import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; import org.apache.hadoop.hdfs.protocol.BlockLocalPathInfo; import org.apache.hadoop.hdfs.protocol.ExtendedBlock; - import org.apache.hadoop.hdfs.protocol.HdfsBlocksMetadata; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; +import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.*; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
