This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-2665-ofs in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 3c5ec6a0346259fc82130729aabc726a5a3d17be Merge: 5616dc3 7d132ce Author: Siyao Meng <[email protected]> AuthorDate: Fri Mar 27 10:21:56 2020 -0700 Merge branch 'master' into HDDS-2665-ofs .../comment-commands/debug.sh | 17 +- .../comment-commands/help.sh | 25 +- .../comment-commands/label.sh | 23 +- .../comment-commands/pending.sh | 27 +- .../comment-commands/ready.sh | 31 +- .../comment-commands/retest.sh | 27 +- .github/process-comment.sh | 56 + .../workflows/comments.yaml | 27 +- .github/workflows/post-commit.yml | 18 +- .github/workflows/pr.yml | 18 +- README.md | 4 +- .../apache/hadoop/hdds/conf/RatisClientConfig.java | 6 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 9 +- .../hadoop/hdds/scm/XceiverClientMetrics.java | 2 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 63 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 108 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 11 +- hadoop-hdds/common/pom.xml | 2 +- .../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 3 + .../org/apache/hadoop/hdds/HddsConfigKeys.java | 9 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 141 +- .../hadoop/hdds/annotation/InterfaceAudience.java | 73 + .../hadoop/hdds/annotation/InterfaceStability.java | 69 + .../hadoop/hdds/annotation}/package-info.java | 7 +- .../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java} | 26 +- .../conf/DatanodeRatisServerConfig.java | 22 +- .../apache/hadoop/hdds/conf/HddsConfServlet.java | 4 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 18 +- .../hadoop/hdds/fs/CachingSpaceUsageSource.java | 4 +- .../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 4 +- .../hadoop/hdds/fs/SpaceUsageCheckParams.java | 4 +- .../hadoop/hdds/fs/SpaceUsagePersistence.java | 4 +- .../apache/hadoop/hdds/fs/SpaceUsageSource.java | 4 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 6 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 122 +- .../org/apache/hadoop/hdds/recon/ReconConfig.java | 93 ++ ...erPlacementPolicy.java => PlacementPolicy.java} | 12 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 29 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 2 +- .../common/helpers/StorageContainerException.java | 2 +- .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 7 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +- .../hadoop/hdds/scm/net/NodeSchemaLoader.java | 4 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 57 +- ...lockLocationProtocolClientSideTranslatorPB.java | 2 +- .../scm/protocolPB/ScmBlockLocationProtocolPB.java | 2 +- ...inerLocationProtocolClientSideTranslatorPB.java | 2 +- .../StorageContainerLocationProtocolPB.java | 2 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 7 +- .../security/token/OzoneBlockTokenIdentifier.java | 2 +- .../security/token/OzoneBlockTokenSelector.java | 2 +- .../x509/certificate/utils/package-info.java | 4 +- .../apache/hadoop/hdds/utils/HddsVersionInfo.java | 4 +- .../org/apache/hadoop/hdds/utils/Scheduler.java | 5 +- .../org/apache/hadoop/hdds/utils/VersionInfo.java | 4 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 41 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 +- .../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 4 +- .../org/apache/hadoop/ozone/audit/SCMAction.java | 1 + .../common/InconsistentStorageStateException.java | 4 +- .../ozone/common/OzoneChecksumException.java | 4 +- .../org/apache/hadoop/ozone/common/Storage.java | 10 +- .../apache/hadoop/ozone/common/StorageInfo.java | 2 +- .../helpers/ContainerCommandRequestPBHelper.java | 4 +- .../org/apache/hadoop/ozone/lock/LockManager.java | 6 +- .../hadoop/ozone/lock/ReadWriteLockable.java | 16 +- .../src/main/proto/DatanodeContainerProtocol.proto | 1 + hadoop-hdds/common/src/main/proto/hdds.proto | 1 + .../common/src/main/resources/ozone-default.xml | 182 ++- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 117 +- .../apache/hadoop/hdds/ratis/TestRatisHelper.java | 53 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 8 +- .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +- .../apache/hadoop/ozone/lock/TestLockManager.java | 30 + .../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +- .../hadoop/ozone/HddsDatanodeHttpServer.java | 2 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 48 +- .../container/common/helpers/ContainerMetrics.java | 2 +- .../container/common/impl/ChunkLayOutVersion.java | 70 +- .../ozone/container/common/impl/ContainerData.java | 41 +- .../container/common/impl/ContainerDataYaml.java | 5 +- .../ozone/container/common/impl/ContainerSet.java | 14 + .../container/common/impl/HddsDispatcher.java | 15 +- .../ozone/container/common/interfaces/Handler.java | 18 +- .../common/interfaces/VolumeChoosingPolicy.java | 2 +- .../report/CommandStatusReportPublisher.java | 2 +- .../common/report/ContainerReportPublisher.java | 2 +- .../common/report/NodeReportPublisher.java | 2 +- .../common/report/PipelineReportPublisher.java | 2 +- .../common/statemachine/DatanodeStateMachine.java | 28 +- .../common/statemachine/EndpointStateMachine.java | 6 +- .../common/statemachine/SCMConnectionManager.java | 66 +- .../common/statemachine/StateContext.java | 60 +- .../CreatePipelineCommandHandler.java | 3 + .../commandhandler/DeleteBlocksCommandHandler.java | 2 +- .../common/states/datanode/InitDatanodeState.java | 8 +- .../states/datanode/RunningDatanodeState.java | 106 +- .../states/endpoint/HeartbeatEndpointTask.java | 8 +- .../states/endpoint/RegisterEndpointTask.java | 59 +- .../states/endpoint/VersionEndpointTask.java | 4 +- .../common/transport/server/ratis/CSMMetrics.java | 2 +- .../server/ratis/ContainerStateMachine.java | 69 +- .../transport/server/ratis/DispatcherContext.java | 4 +- .../transport/server/ratis/XceiverServerRatis.java | 106 +- .../container/common/volume/AsyncChecker.java | 4 +- .../ozone/container/common/volume/HddsVolume.java | 26 +- .../container/common/volume/HddsVolumeChecker.java | 50 +- .../common/volume/ImmutableVolumeSet.java | 49 +- .../{VolumeSet.java => MutableVolumeSet.java} | 78 +- .../common/volume/ThrottledAsyncChecker.java | 8 +- .../ozone/container/common/volume/VolumeSet.java | 518 +----- .../container/keyvalue/KeyValueBlockIterator.java | 2 +- .../container/keyvalue/KeyValueContainerData.java | 25 +- .../ozone/container/keyvalue/KeyValueHandler.java | 60 +- .../container/keyvalue/helpers/ChunkUtils.java | 262 +-- .../keyvalue/impl/ChunkManagerDispatcher.java | 148 ++ .../keyvalue/impl/ChunkManagerDummyImpl.java | 115 +- .../keyvalue/impl/ChunkManagerFactory.java | 45 +- .../keyvalue/impl/FilePerBlockStrategy.java | 297 ++++ ...kManagerImpl.java => FilePerChunkStrategy.java} | 195 ++- .../keyvalue/interfaces/ChunkManager.java | 15 +- .../background/BlockDeletingService.java | 22 +- .../container/ozoneimpl/ContainerController.java | 11 +- .../ozoneimpl/ContainerDataScrubberMetrics.java | 2 +- .../ContainerMetadataScrubberMetrics.java | 2 +- .../ozone/container/ozoneimpl/ContainerReader.java | 6 +- .../ozoneimpl/ContainerScrubberConfiguration.java | 2 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 33 +- .../replication/DownloadAndImportReplicator.java | 8 +- .../ozone/protocol/ReconDatanodeProtocol.java} | 29 +- .../protocol/StorageContainerDatanodeProtocol.java | 2 +- .../protocol/StorageContainerNodeProtocol.java | 2 +- .../ozone/protocolPB/ReconDatanodeProtocolPB.java} | 24 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 1 + .../proto/StorageContainerDatanodeProtocol.proto | 1 + .../ozone/container/common/ContainerTestUtils.java | 59 + .../ozone/container/common/SCMTestUtils.java | 2 +- .../container/common/TestBlockDeletingService.java | 182 +-- .../container/common/TestChunkLayOutVersion.java | 21 +- .../common/TestKeyValueContainerData.java | 2 + .../container/common/helpers/TestBlockData.java | 6 +- .../common/impl/TestContainerDataYaml.java | 10 +- .../impl/TestContainerDeletionChoosingPolicy.java | 2 + .../common/impl/TestContainerPersistence.java | 69 +- .../container/common/impl/TestContainerSet.java | 8 +- .../container/common/impl/TestHddsDispatcher.java | 50 +- .../container/common/interfaces/TestHandler.java | 3 +- .../common/statemachine/TestStateContext.java | 9 +- .../TestCloseContainerCommandHandler.java | 4 +- .../states/endpoint/TestHeartbeatEndpointTask.java | 6 +- .../common/volume/TestHddsVolumeChecker.java | 101 +- .../container/common/volume/TestVolumeSet.java | 12 +- .../common/volume/TestVolumeSetDiskChecks.java | 11 +- .../keyvalue/TestKeyValueBlockIterator.java | 8 +- .../container/keyvalue/TestKeyValueContainer.java | 10 +- .../keyvalue/TestKeyValueContainerCheck.java | 15 +- .../TestKeyValueContainerMarkUnhealthy.java | 5 +- .../container/keyvalue/TestKeyValueHandler.java | 7 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 4 +- .../container/keyvalue/TestTarContainerPacker.java | 4 +- .../container/keyvalue/helpers/TestChunkUtils.java | 48 +- .../keyvalue/{ => impl}/TestBlockManagerImpl.java | 10 +- .../TestChunkManager.java} | 51 +- .../container/ozoneimpl/TestOzoneContainer.java | 11 +- .../replication/TestReplicationSupervisor.java | 10 +- .../docs/content/beyond/DockerCheatSheet.zh.md | 85 + hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 2 +- .../docs/content/beyond/RunningWithHDFS.zh.md | 64 + hadoop-hdds/docs/content/concept/Hdds.zh.md | 40 + hadoop-hdds/docs/content/concept/Overview.zh.md | 50 + .../docs/content/concept/OzoneManager.zh.md | 64 + hadoop-hdds/docs/content/gdpr/GDPR in Ozone.zh.md | 36 + hadoop-hdds/framework/pom.xml | 2 +- .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 2 +- .../apache/hadoop/hdds/protocol}/package-info.java | 7 +- .../SCMSecurityProtocolClientSideTranslatorPB.java | 0 .../hdds/protocolPB/SCMSecurityProtocolPB.java | 0 .../hadoop/hdds/protocolPB/package-info.java | 0 .../hdds/security/token/BlockTokenException.java | 0 .../hdds/security/token/BlockTokenVerifier.java | 0 .../hadoop/hdds/security/token/TokenVerifier.java | 0 .../hadoop/hdds/security/token/package-info.java | 2 +- .../x509/certificate/authority/BaseApprover.java | 0 .../certificate/authority/CertificateApprover.java | 0 .../certificate/authority/CertificateServer.java | 0 .../certificate/authority/CertificateStore.java | 0 .../certificate/authority/DefaultApprover.java | 0 .../certificate/authority/DefaultCAServer.java | 0 .../authority/PKIProfiles/DefaultCAProfile.java | 0 .../authority/PKIProfiles/DefaultProfile.java | 2 +- .../authority/PKIProfiles/PKIProfile.java | 0 .../authority/PKIProfiles/package-info.java | 0 .../x509/certificate/authority/package-info.java | 0 .../x509/certificate/client/CertificateClient.java | 0 .../certificate/client/DNCertificateClient.java | 0 .../client/DefaultCertificateClient.java | 60 +- .../certificate/client/OMCertificateClient.java | 0 .../x509/certificate/client/package-info.java | 0 .../security/x509/certificate/utils/CRLCodec.java | 0 .../x509/certificate/utils/package-info.java | 0 .../certificates/utils/CertificateSignRequest.java | 0 .../certificates/utils/SelfSignedCertificate.java | 0 .../x509/certificates/utils/package-info.java | 0 .../x509/exceptions/CertificateException.java | 0 .../security/x509/exceptions/package-info.java | 0 .../hdds/security/x509/keys/HDDSKeyGenerator.java | 0 .../hadoop/hdds/security/x509/keys/KeyCodec.java | 0 .../hdds/security/x509/keys/SecurityUtil.java | 0 .../hdds/security/x509/keys/package-info.java | 0 .../hadoop/hdds/security/x509/package-info.java | 99 ++ .../server/OzoneProtocolMessageDispatcher.java | 2 +- .../org/apache/hadoop/hdds/server/ServerUtils.java | 11 +- .../hdds/server/http/AdminAuthorizedServlet.java} | 33 +- .../hdds/server/{ => http}/BaseHttpServer.java | 173 +- .../hadoop/hdds/server/http/FilterContainer.java | 32 +- .../hadoop/hdds/server/http/FilterInitializer.java | 16 +- .../hadoop/hdds/server/http/HtmlQuoting.java | 235 +++ .../apache/hadoop/hdds/server/http/HttpConfig.java | 75 + .../hadoop/hdds/server/http/HttpRequestLog.java | 107 ++ .../hdds/server/http/HttpRequestLogAppender.java} | 45 +- .../hadoop/hdds/server/http/HttpServer2.java | 1706 ++++++++++++++++++++ .../hdds/server/{ => http}/LogStreamServlet.java | 2 +- .../hadoop/hdds/server/http/NoCacheFilter.java | 55 + .../hdds/server/{ => http}/ProfileServlet.java | 4 +- .../server/{ => http}/PrometheusMetricsSink.java | 2 +- .../hdds/server/{ => http}/PrometheusServlet.java | 2 +- .../server/{ => http}/RatisDropwizardExports.java | 2 +- .../{ => http}/RatisNameRewriteSampleBuilder.java | 2 +- .../hdds/server/http/StaticUserWebFilter.java | 157 ++ .../hadoop/hdds/server/http/package-info.java | 10 +- .../apache/hadoop/hdds/utils}/HddsServerUtil.java | 170 +- .../org/apache/hadoop/hdds/utils/LevelDBStore.java | 12 +- .../hadoop/hdds/utils/LevelDBStoreIterator.java | 0 .../hadoop/hdds/utils/MetaStoreIterator.java | 0 .../hadoop/hdds/utils/MetadataKeyFilters.java | 0 .../apache/hadoop/hdds/utils/MetadataStore.java | 2 +- .../hadoop/hdds/utils/MetadataStoreBuilder.java | 0 .../hadoop/hdds/utils}/ProtocolMessageMetrics.java | 2 +- .../org/apache/hadoop/hdds/utils/RocksDBStore.java | 1 + .../hadoop/hdds/utils/RocksDBStoreIterator.java | 0 .../hadoop/hdds/utils/RocksDBStoreMBean.java | 0 .../hadoop/hdds/utils/db/BatchOperation.java | 0 .../hadoop/hdds/utils/db/ByteArrayCodec.java | 0 .../hadoop/hdds/utils/db/ByteArrayKeyValue.java | 0 .../org/apache/hadoop/hdds/utils/db/Codec.java | 0 .../apache/hadoop/hdds/utils/db/CodecRegistry.java | 0 .../apache/hadoop/hdds/utils/db/DBCheckpoint.java | 0 .../hadoop/hdds/utils/db/DBConfigFromFile.java | 0 .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 5 +- .../org/apache/hadoop/hdds/utils/db/DBStore.java | 2 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 70 +- .../hadoop/hdds/utils/db/DBUpdatesWrapper.java | 0 .../apache/hadoop/hdds/utils/db/IntegerCodec.java | 0 .../org/apache/hadoop/hdds/utils/db/LongCodec.java | 0 .../hadoop/hdds/utils/db/RDBBatchOperation.java | 0 .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 4 +- .../apache/hadoop/hdds/utils/db/RDBMetrics.java | 28 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 0 .../hadoop/hdds/utils/db/RDBStoreIterator.java | 0 .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 28 +- .../hadoop/hdds/utils/db/RocksDBCheckpoint.java | 4 +- .../hadoop/hdds/utils/db/RocksDBConfiguration.java | 0 .../utils/db/SequenceNumberNotFoundException.java | 0 .../apache/hadoop/hdds/utils/db/StringCodec.java | 0 .../org/apache/hadoop/hdds/utils/db/Table.java | 19 +- .../apache/hadoop/hdds/utils/db/TableConfig.java | 0 .../apache/hadoop/hdds/utils/db/TableIterator.java | 0 .../apache/hadoop/hdds/utils/db/TypedTable.java | 24 + .../hadoop/hdds/utils/db/cache/CacheKey.java | 0 .../hadoop/hdds/utils/db/cache/CacheResult.java | 0 .../hadoop/hdds/utils/db/cache/CacheValue.java | 0 .../hadoop/hdds/utils/db/cache/EpochEntry.java | 0 .../hadoop/hdds/utils/db/cache/TableCache.java | 4 +- .../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 4 +- .../hadoop/hdds/utils/db/cache/package-info.java | 0 .../apache/hadoop/hdds/utils/db/package-info.java | 0 .../apache/hadoop/hdds}/utils/package-info.java | 10 +- .../token/TestOzoneBlockTokenIdentifier.java | 0 .../hadoop/hdds/security/token/package-info.java | 0 .../x509/certificate/authority/MockApprover.java | 0 .../x509/certificate/authority/MockCAStore.java | 0 .../certificate/authority/TestDefaultCAServer.java | 0 .../certificate/authority/TestDefaultProfile.java | 0 .../x509/certificate/authority/package-info.java | 0 .../client/TestCertificateClientInit.java | 0 .../client/TestDefaultCertificateClient.java | 0 .../x509/certificate/utils/TestCRLCodec.java | 0 .../certificate/utils/TestCertificateCodec.java | 0 .../x509/certificate/utils/package-info.java | 0 .../certificates/TestCertificateSignRequest.java | 0 .../x509/certificates/TestRootCertificate.java | 0 .../security/x509/certificates/package-info.java | 0 .../security/x509/keys/TestHDDSKeyGenerator.java | 0 .../hdds/security/x509/keys/TestKeyCodec.java | 0 .../hdds/security/x509/keys/package-info.java | 0 .../hadoop/hdds/security/x509/package-info.java | 0 .../hdds/server/{ => http}/TestBaseHttpServer.java | 3 +- .../hadoop/hdds/server/http/TestHtmlQuoting.java | 96 ++ .../hdds/server/http/TestHttpRequestLog.java | 51 + .../server/http/TestHttpRequestLogAppender.java | 21 +- .../hdds/server/{ => http}/TestProfileServlet.java | 6 +- .../{ => http}/TestPrometheusMetricsSink.java | 2 +- .../{ => http}/TestRatisDropwizardExports.java | 7 +- .../server/{ => http}/TestRatisNameRewrite.java | 2 +- .../hadoop/hdds/server/http/package-info.java | 10 +- .../hadoop/hdds/utils/TestMetadataStore.java | 0 .../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 0 .../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 0 .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 0 .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 0 .../hadoop/hdds/utils/db/TestRDBTableStore.java | 63 + .../hdds/utils/db/TestTypedRDBTableStore.java | 18 + .../hdds/utils/db/cache/TestTableCacheImpl.java | 0 .../hadoop/hdds/utils/db/cache/package-info.java | 0 .../apache/hadoop/hdds/utils/db/package-info.java | 0 .../apache/hadoop/hdds/utils/package-info.java} | 31 +- .../src/test/resources/test.db.ini | 0 ...onPolicy.java => SCMCommonPlacementPolicy.java} | 23 +- .../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 12 +- .../hdds/scm/container/ReplicationManager.java | 27 +- .../hdds/scm/container/SCMContainerManager.java | 39 +- .../ContainerPlacementPolicyFactory.java | 22 +- .../algorithms/SCMContainerPlacementCapacity.java | 4 +- .../algorithms/SCMContainerPlacementRackAware.java | 12 +- .../algorithms/SCMContainerPlacementRandom.java | 6 +- .../scm/container/states/ContainerStateMap.java | 3 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 14 + .../hadoop/hdds/scm/node/NodeManagerMXBean.java | 2 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 18 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 19 + .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 2 +- .../hdds/scm/node/SCMNodeStorageStatMXBean.java | 2 +- .../hdds/scm/node/states/Node2ObjectsMap.java | 4 +- .../hdds/scm/node/states/Node2PipelineMap.java | 11 +- .../hadoop/hdds/scm/node/states/NodeStateMap.java | 2 +- .../scm/pipeline/BackgroundPipelineCreator.java | 9 + .../hadoop/hdds/scm/pipeline/PipelineManager.java | 13 +- .../hdds/scm/pipeline/PipelineManagerMXBean.java | 2 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 428 +++++ .../hdds/scm/pipeline/PipelineReportHandler.java | 9 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 5 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 164 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 117 ++ .../hdds/scm/pipeline/SCMPipelineManager.java | 131 +- .../hdds/scm/pipeline/SCMPipelineMetrics.java | 35 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 2 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 2 +- ...lockLocationProtocolServerSideTranslatorPB.java | 4 +- ...inerLocationProtocolServerSideTranslatorPB.java | 30 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 13 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 4 - .../hadoop/hdds/scm/safemode/SafeModeHandler.java | 87 +- .../hdds/scm/safemode/SafeModeNotification.java | 25 + .../scm/server/OzoneStorageContainerManager.java | 12 + .../hdds/scm/server/SCMBlockProtocolServer.java | 4 +- .../hdds/scm/server/SCMClientProtocolServer.java | 27 +- .../hdds/scm/server/SCMContainerMetrics.java | 2 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 87 +- .../apache/hadoop/hdds/scm/server/SCMMXBean.java | 2 +- .../hadoop/hdds/scm/server/SCMPolicyProvider.java | 8 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 6 +- .../hdds/scm/server/StorageContainerManager.java | 63 +- .../server/StorageContainerManagerHttpServer.java | 2 +- .../scm/server/StorageContainerManagerStarter.java | 9 +- ...ServerUtilTest.java => TestHddsServerUtil.java} | 114 +- .../hadoop/hdds/scm/TestHddsServerUtils.java | 1 + .../scm/TestStorageContainerManagerHttpServer.java | 8 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 65 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 54 +- .../container/TestCloseContainerEventHandler.java | 7 +- .../hdds/scm/container/TestReplicationManager.java | 7 +- .../algorithms/TestContainerPlacementFactory.java | 7 +- .../hdds/scm/node/TestContainerPlacement.java | 5 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 13 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 2 + .../scm/pipeline/MockRatisPipelineProvider.java | 43 + .../TestPipelineDatanodesIntersection.java | 133 ++ .../scm/pipeline/TestPipelinePlacementPolicy.java | 340 ++++ .../scm/pipeline/TestRatisPipelineProvider.java | 75 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 74 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 19 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 6 +- .../hdds/scm/safemode/TestSafeModeHandler.java | 11 +- .../scm/server/TestSCMBlockProtocolServer.java | 2 +- .../scm/server/TestSCMClientProtocolServer.java | 6 +- .../testutils/ReplicationNodeManagerMock.java | 16 + hadoop-hdds/tools/pom.xml | 19 + .../hdds/scm/cli}/ContainerOperationClient.java | 48 +- .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 13 +- .../DatanodeCommands.java} | 24 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 121 ++ .../hadoop/hdds/scm/cli/datanode/package-info.java | 9 +- ...bcommand.java => CreatePipelineSubcommand.java} | 65 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 11 +- .../hdds/scm/cli/pipeline/PipelineCommands.java | 1 + .../apache/hadoop/ozone/client/OzoneBucket.java | 8 +- .../hadoop/ozone/client/OzoneClientFactory.java | 52 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 7 +- .../ozone/client/io/BlockOutputStreamEntry.java | 2 +- .../client/io/BlockOutputStreamEntryPool.java | 23 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 69 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 11 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 13 +- .../ozone/OzoneIllegalArgumentException.java | 4 +- .../hadoop/ozone/om/exceptions/OMException.java | 4 + .../ozone/om/ha/OMFailoverProxyProvider.java | 18 +- .../org/apache/hadoop/ozone/om/ha/OMProxyInfo.java | 25 +- .../ozone/om/helpers/EncryptionBucketInfo.java | 4 +- .../hadoop/ozone/om/helpers/OMRatisHelper.java | 21 + .../hadoop/ozone/om/helpers/OmBucketInfo.java | 64 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 75 +- .../ozone/om/helpers/OmMultipartKeyInfo.java | 35 +- .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 22 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 59 +- .../hadoop/ozone/om/helpers/OzoneFileStatus.java | 16 +- .../hadoop/ozone/om/helpers/WithObjectID.java | 123 ++ ...OzoneManagerProtocolClientSideTranslatorPB.java | 63 +- .../om/protocolPB/OzoneManagerProtocolPB.java | 2 +- .../security/OzoneBlockTokenSecretManager.java | 4 +- .../OzoneDelegationTokenSecretManager.java | 15 +- .../security/OzoneDelegationTokenSelector.java | 2 +- .../hadoop/ozone/security/OzoneSecretKey.java | 4 +- .../hadoop/ozone/security/OzoneSecretManager.java | 4 +- .../hadoop/ozone/security/OzoneSecretStore.java | 2 +- .../ozone/security/OzoneTokenIdentifier.java | 4 +- .../ozone/security/acl/IAccessAuthorizer.java | 4 +- .../hadoop/ozone/security/acl/OzoneAclConfig.java | 3 + .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 4 +- .../apache/hadoop/ozone/web/utils/OzoneUtils.java | 2 +- .../src/main/proto/OzoneManagerProtocol.proto | 7 + .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +- hadoop-ozone/csi/pom.xml | 2 +- .../dev-support/checks/_mvn_unit_report.sh | 36 +- hadoop-ozone/dev-support/checks/integration.sh | 7 +- hadoop-ozone/dev-support/checks/unit.sh | 12 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 18 +- .../{Datanode.xml => Datanode1.xml} | 6 +- .../{Datanode.xml => Datanode2.xml} | 6 +- .../{Datanode.xml => Datanode3.xml} | 6 +- .../dist/src/main/assemblies/ozone-src.xml | 4 +- .../dist/src/main/compose/ozone-om-ha-s3/test.sh | 4 + .../compose/ozone-topology/docker-compose.yaml | 28 + .../{ozone/test.sh => ozone-topology/hdds-3084.sh} | 27 +- .../dist/src/main/compose/ozone/docker-config | 3 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 + .../src/main/compose/ozonesecure-mr/docker-config | 2 + .../src/main/compose/ozonesecure/docker-config | 8 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 48 +- .../dist/src/main/smoketest/commonlib.robot | 1 + .../dist/src/main/smoketest/freon/freon.robot | 36 + .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 2 + .../dist/src/main/smoketest/recon/recon-api.robot | 45 +- .../dist/src/main/smoketest/s3/webui.robot | 3 +- .../scmcli/{pipeline.robot => datanode.robot} | 7 +- .../dist/src/main/smoketest/scmcli/pipeline.robot | 7 +- .../pipeline.robot => topology/loaddata.robot} | 10 +- .../pipeline.robot => topology/readdata.robot} | 7 +- hadoop-ozone/dist/src/shell/ozone/ozone | 5 + .../mini-chaos-tests/src/test/bin/start-chaos.sh | 7 +- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 19 +- .../hadoop/ozone/MiniOzoneLoadGenerator.java | 265 +-- .../hadoop/ozone/TestMiniChaosOzoneCluster.java | 33 +- .../ozone/loadgenerators/AgedLoadGenerator.java | 98 ++ .../hadoop/ozone/loadgenerators/DataBuffer.java | 53 + .../loadgenerators/FilesystemLoadGenerator.java | 69 + .../hadoop/ozone/loadgenerators/LoadExecutors.java | 101 ++ .../ozone/loadgenerators/LoadGenerator.java} | 24 +- .../ozone/loadgenerators/RandomLoadGenerator.java | 68 + .../org/apache/hadoop/ozone/utils/LoadBucket.java | 23 +- .../src/test/resources/log4j.properties | 10 +- hadoop-ozone/insight/pom.xml | 6 - .../hadoop/ozone/insight/BaseInsightPoint.java | 4 +- ...gSubcommandTest.java => TestLogSubcommand.java} | 14 +- hadoop-ozone/integration-test/pom.xml | 113 -- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 214 ++- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 46 + .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 7 +- .../hadoop/fs/ozone/contract/OzoneContract.java | 23 + .../hadoop/hdds/scm/TestRatisPipelineLeader.java | 65 +- .../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 6 +- .../hdds/scm/pipeline/TestPipelineClose.java | 1 + .../TestRatisPipelineCreateAndDestroy.java | 24 +- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 5 +- .../safemode/TestSCMSafeModeWithPipelineRules.java | 3 + .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 50 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 131 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 34 +- .../org/apache/hadoop/ozone/RatisTestHelper.java | 4 +- .../hadoop/ozone/TestContainerOperations.java | 9 +- .../TestContainerStateMachineIdempotency.java | 5 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 19 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 100 +- .../ozone/client/rpc/Test2WayCommitInRatis.java | 56 +- .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 2 +- .../ozone/client/rpc/TestBlockOutputStream.java | 23 +- .../rpc/TestBlockOutputStreamWithFailures.java | 428 +---- .../rpc/TestCloseContainerHandlingByClient.java | 3 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 84 +- .../rpc/TestContainerReplicationEndToEnd.java | 14 +- .../client/rpc/TestContainerStateMachine.java | 11 +- .../TestContainerStateMachineFailureOnRead.java | 224 +++ .../rpc/TestContainerStateMachineFailures.java | 2 +- .../client/rpc/TestDeleteWithSlowFollower.java | 57 +- .../client/rpc/TestFailureHandlingByClient.java | 31 +- .../client/rpc/TestHybridPipelineOnDatanode.java | 5 +- .../ozone/client/rpc/TestKeyInputStream.java | 3 +- .../rpc/TestMultiBlockWritesWithDnFailures.java | 31 +- .../rpc/TestOzoneClientRetriesOnException.java | 74 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 14 +- .../rpc/TestOzoneRpcClientForAclAuditLog.java | 3 +- .../ozone/client/rpc/TestWatchForCommit.java | 5 +- .../TestCloseContainerByPipeline.java | 7 +- .../commandhandler/TestCloseContainerHandler.java | 2 +- .../commandhandler/TestDeleteContainerHandler.java | 2 +- .../container/metrics/TestContainerMetrics.java | 3 +- .../container/ozoneimpl/TestOzoneContainer.java | 2 +- .../ozoneimpl/TestOzoneContainerRatis.java | 4 +- .../container/ozoneimpl/TestRatisManager.java | 6 +- .../container/server/TestContainerServer.java | 3 +- .../server/TestSecureContainerServer.java | 3 +- .../ozone/dn/ratis/TestDnRatisLogParser.java | 85 + .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 4 +- .../hadoop/ozone/freon/TestDataValidate.java | 55 +- .../freon/TestDataValidateWithDummyContainers.java | 18 +- .../ozone/freon/TestFreonWithDatanodeRestart.java | 35 +- .../ozone/freon/TestFreonWithPipelineDestroy.java | 1 + .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 21 + .../ozone/om/TestContainerReportWithKeys.java | 2 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 + .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 24 +- .../hadoop/ozone/om/TestOzoneManagerRestart.java | 3 + .../ozone/om/parser/TestOMRatisLogParser.java | 124 ++ .../hadoop/ozone/ozShell/TestOzoneShellHA.java | 143 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 200 +++ ...stRecon.java => TestReconWithOzoneManager.java} | 89 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 142 ++ .../hadoop/ozone/scm/TestContainerSmallFile.java | 4 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 7 +- .../TestSCMContainerPlacementPolicyMetrics.java | 1 + .../hadoop/ozone/scm/node/TestQueryNode.java | 5 +- .../TestSCMPipelineBytesWrittenMetrics.java | 138 ++ .../src/test/resources/ozone-site.xml | 5 + .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 171 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 2 +- .../java/org/apache/hadoop/ozone/om/OMMXBean.java | 2 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 2 +- .../apache/hadoop/ozone/om/OMPolicyProvider.java | 8 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 45 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 182 ++- .../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 +- .../hadoop/ozone/om/OzoneManagerStarter.java | 9 +- .../apache/hadoop/ozone/om/PrefixManagerImpl.java | 42 +- .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 6 +- .../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 3 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 14 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +- .../metrics/OzoneManagerDoubleBufferMetrics.java | 49 +- .../hadoop/ozone/om/request/OMClientRequest.java | 46 +- .../om/request/bucket/OMBucketCreateRequest.java | 17 +- .../om/request/bucket/OMBucketDeleteRequest.java | 10 +- .../request/bucket/OMBucketSetPropertyRequest.java | 9 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 13 +- .../om/request/file/OMDirectoryCreateRequest.java | 271 +++- .../ozone/om/request/file/OMFileCreateRequest.java | 150 +- .../ozone/om/request/file/OMFileRequest.java | 153 +- .../om/request/key/OMAllocateBlockRequest.java | 12 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 94 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 22 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 14 +- .../ozone/om/request/key/OMKeyPurgeRequest.java | 6 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 37 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 25 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 65 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 45 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 47 +- .../om/request/key/acl/OMKeySetAclRequest.java | 40 +- .../request/key/acl/prefix/OMPrefixAclRequest.java | 94 +- .../key/acl/prefix/OMPrefixAddAclRequest.java | 51 +- .../key/acl/prefix/OMPrefixRemoveAclRequest.java | 48 +- .../key/acl/prefix/OMPrefixSetAclRequest.java | 44 +- .../request/s3/bucket/S3BucketCreateRequest.java | 13 +- .../request/s3/bucket/S3BucketDeleteRequest.java | 10 +- .../S3InitiateMultipartUploadRequest.java | 74 +- .../multipart/S3MultipartUploadAbortRequest.java | 102 +- .../S3MultipartUploadCommitPartRequest.java | 100 +- .../S3MultipartUploadCompleteRequest.java | 75 +- .../om/request/s3/security/S3GetSecretRequest.java | 6 +- .../security/OMCancelDelegationTokenRequest.java | 7 +- .../security/OMGetDelegationTokenRequest.java | 16 +- .../security/OMRenewDelegationTokenRequest.java | 7 +- .../om/request/volume/OMVolumeCreateRequest.java | 16 +- .../om/request/volume/OMVolumeDeleteRequest.java | 10 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 13 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 13 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 71 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 37 +- .../volume/acl/OMVolumeRemoveAclRequest.java | 37 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 35 +- .../response/file/OMDirectoryCreateResponse.java | 37 +- .../om/response/file/OMFileCreateResponse.java | 12 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 45 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 29 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 7 +- .../om/response/key/acl/OMKeyAclResponse.java | 29 +- .../key/acl/prefix/OMPrefixAclResponse.java | 46 +- .../S3InitiateMultipartUploadResponse.java | 39 +- .../multipart/S3MultipartUploadAbortResponse.java | 62 +- .../S3MultipartUploadCommitPartResponse.java | 35 +- .../S3MultipartUploadCompleteResponse.java | 64 +- .../om/response/volume/OMVolumeAclOpResponse.java | 28 +- .../om/response/volume/OMVolumeCreateResponse.java | 2 +- .../om/response/volume/OMVolumeDeleteResponse.java | 2 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 43 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 1 + .../protocolPB/OzoneManagerRequestHandler.java | 2 + .../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +- .../web/ozShell/bucket/AddAclBucketHandler.java | 33 +- .../web/ozShell/bucket/GetAclBucketHandler.java | 4 +- .../web/ozShell/bucket/RemoveAclBucketHandler.java | 33 +- .../web/ozShell/bucket/SetAclBucketHandler.java | 31 +- .../ozone/web/ozShell/keys/AddAclKeyHandler.java | 33 +- .../ozone/web/ozShell/keys/GetAclKeyHandler.java | 4 +- .../ozone/web/ozShell/keys/ListKeyHandler.java | 3 +- .../web/ozShell/keys/RemoveAclKeyHandler.java | 33 +- .../ozone/web/ozShell/keys/SetAclKeyHandler.java | 31 +- .../ozone/web/ozShell/s3/GetS3SecretHandler.java | 11 +- .../ozShell/{bucket => s3}/S3BucketMapping.java | 8 +- .../hadoop/ozone/web/ozShell/s3/S3Handler.java | 38 +- .../hadoop/ozone/web/ozShell/s3/S3Shell.java | 1 - .../web/ozShell/volume/AddAclVolumeHandler.java | 33 +- .../web/ozShell/volume/GetAclVolumeHandler.java | 4 +- .../web/ozShell/volume/RemoveAclVolumeHandler.java | 33 +- .../web/ozShell/volume/SetAclVolumeHandler.java | 33 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 48 + .../ozone/om/TestOzoneManagerHttpServer.java | 8 +- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 7 + ...TestOzoneManagerDoubleBufferWithOMResponse.java | 182 +-- .../ozone/om/request/TestOMRequestUtils.java | 3 +- .../bucket/TestOMBucketSetPropertyRequest.java | 4 +- .../request/file/TestOMDirectoryCreateRequest.java | 2 +- .../om/request/file/TestOMFileCreateRequest.java | 40 +- .../ozone/om/request/key/TestOMKeyAclRequest.java | 92 ++ .../om/request/key/TestOMKeyCommitRequest.java | 42 +- .../om/request/key/TestOMPrefixAclRequest.java | 101 ++ .../request/volume/TestOMVolumeCreateRequest.java | 20 +- .../volume/acl/TestOMVolumeAddAclRequest.java | 33 + .../volume/acl/TestOMVolumeRemoveAclRequest.java | 47 +- .../volume/acl/TestOMVolumeSetAclRequest.java | 35 + .../file/TestOMDirectoryCreateResponse.java | 37 +- .../om/response/key/TestOMKeyCommitResponse.java | 11 +- .../om/response/key/TestOMKeyCreateResponse.java | 4 +- .../om/response/key/TestOMKeyDeleteResponse.java | 6 +- .../s3/multipart/TestS3MultipartResponse.java | 8 +- hadoop-ozone/ozonefs-lib-legacy/pom.xml | 5 +- .../java/org/apache/hadoop/fs/ozone/BasicOzFs.java | 4 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 92 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 48 +- .../apache/hadoop/fs/ozone/FileStatusAdapter.java | 12 +- .../hadoop/fs/ozone/FilteredClassLoader.java | 1 + .../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 4 +- .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 6 +- .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 4 +- .../hadoop/fs/ozone/OzoneFSStorageStatistics.java | 4 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +- .../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 2 +- .../org/apache/hadoop/fs/ozone/package-info.java | 4 +- .../fs/ozone/TestOzoneFileSystemWithMocks.java | 120 +- .../hadoop/fs/ozone/TestReadWriteStatistics.java | 4 +- .../recon/codegen/ReconSchemaGenerationModule.java | 4 +- ...inition.java => ReconTaskSchemaDefinition.java} | 6 +- .../recon/schema/UtilizationSchemaDefinition.java | 13 + hadoop-ozone/recon/pom.xml | 29 + .../hadoop/ozone/recon/ConfigurationProvider.java | 16 + .../apache/hadoop/ozone/recon/ReconConstants.java | 4 +- .../hadoop/ozone/recon/ReconControllerModule.java | 48 +- .../recon/ReconGuiceServletContextListener.java | 2 +- .../apache/hadoop/ozone/recon/ReconHttpServer.java | 6 +- .../hadoop/ozone/recon/ReconRestServletModule.java | 7 +- .../hadoop/ozone/recon/ReconSchemaManager.java | 2 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 106 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 11 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 5 +- .../ozone/recon/api/ClusterStateEndpoint.java | 110 ++ ...ainerKeyService.java => ContainerEndpoint.java} | 56 +- .../hadoop/ozone/recon/api/NodeEndpoint.java | 129 ++ .../hadoop/ozone/recon/api/PipelineEndpoint.java | 107 ++ .../hadoop/ozone/recon/api/TaskStatusService.java | 7 +- .../hadoop/ozone/recon/api/UtilizationService.java | 14 +- .../hadoop/ozone/recon/api/package-info.java | 2 +- .../recon/api/types/ClusterStateResponse.java | 198 +++ .../ozone/recon/api/types/ContainerKeyPrefix.java | 2 +- .../ozone/recon/api/types/ContainerMetadata.java | 2 +- .../ozone/recon/api/types/ContainersResponse.java | 13 +- .../ozone/recon/api/types/DatanodeMetadata.java | 88 + .../ozone/recon/api/types/DatanodePipeline.java | 39 +- .../types/DatanodeStorageReport.java} | 35 +- .../ozone/recon/api/types/DatanodesResponse.java | 58 + .../hadoop/ozone/recon/api/types/KeyMetadata.java | 2 +- .../hadoop/ozone/recon/api/types/KeysResponse.java | 13 +- ...Metadata.java => MissingContainerMetadata.java} | 46 +- .../recon/api/types/MissingContainersResponse.java | 53 + .../ozone/recon/api/types/PipelineMetadata.java | 218 +++ .../ozone/recon/api/types/PipelinesResponse.java | 59 + .../hadoop/ozone/recon/api/types/package-info.java | 2 +- .../ozone/recon/fsck/MissingContainerInfo.java | 68 + .../ozone/recon/fsck/MissingContainerTask.java | 109 ++ .../recon/{persistence => fsck}/package-info.java | 4 +- .../apache/hadoop/ozone/recon/package-info.java | 2 +- .../recon/persistence/DataSourceConfiguration.java | 5 +- .../persistence/DefaultDataSourceProvider.java | 5 +- .../recon/persistence/JooqPersistenceModule.java | 3 +- .../TransactionalMethodInterceptor.java | 2 +- .../ozone/recon/persistence/package-info.java | 2 +- .../recon/recovery/ReconOMMetadataManager.java | 2 +- .../recon/recovery/ReconOmMetadataManagerImpl.java | 5 +- .../hadoop/ozone/recon/recovery/package-info.java | 2 +- .../hadoop/ozone/recon/scm/PipelineSyncTask.java | 73 + .../ozone/recon/scm/ReconContainerManager.java | 55 +- .../recon/scm/ReconContainerReportHandler.java | 74 + .../recon/scm/ReconDatanodeProtocolServer.java | 31 +- .../ReconIncrementalContainerReportHandler.java | 28 +- .../ozone/recon/scm/ReconNewNodeHandler.java | 53 + .../hadoop/ozone/recon/scm/ReconNodeManager.java | 168 ++ .../ozone/recon/scm/ReconPipelineFactory.java | 19 +- .../ozone/recon/scm/ReconPipelineManager.java | 32 +- .../recon/scm/ReconPipelineReportHandler.java | 2 +- .../ozone/recon/scm/ReconPolicyProvider.java | 66 + .../ozone/recon/scm/ReconSafeModeManager.java | 2 +- .../hadoop/ozone/recon/scm/ReconScmTask.java | 106 ++ .../hadoop/ozone/recon/scm/ReconStorageConfig.java | 2 +- .../scm/ReconStorageContainerManagerFacade.java | 71 +- .../hadoop/ozone/recon/scm/package-info.java | 2 +- .../recon/spi/ContainerDBServiceProvider.java | 13 +- .../recon/spi/HddsDatanodeServiceProvider.java | 5 +- .../recon/spi/OzoneManagerServiceProvider.java | 5 +- .../recon/spi/StorageContainerServiceProvider.java | 5 +- .../spi/impl/ContainerDBServiceProviderImpl.java | 15 +- .../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 100 +- .../recon/spi/impl/ReconContainerDBProvider.java | 2 +- .../impl/StorageContainerServiceProviderImpl.java | 2 +- .../hadoop/ozone/recon/spi/impl/package-info.java | 2 +- .../hadoop/ozone/recon/spi/package-info.java | 2 +- .../ozone/recon/tasks/ContainerKeyMapperTask.java | 15 +- .../ozone/recon/tasks/FileSizeCountTask.java | 73 +- .../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 38 +- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 52 +- .../ozone/recon/tasks/OMUpdateEventBatch.java | 2 +- .../{ReconDBUpdateTask.java => ReconOmTask.java} | 4 +- .../ozone/recon/tasks/ReconTaskController.java | 6 +- .../ozone/recon/tasks/ReconTaskControllerImpl.java | 64 +- .../hadoop/ozone/recon/tasks/package-info.java | 2 +- .../webapps/recon/ozone-recon-web/LICENSE | 32 + .../webapps/recon/ozone-recon-web/api/db.json | 315 ++-- .../webapps/recon/ozone-recon-web/package.json | 6 +- .../webapps/recon/ozone-recon-web/src/App.less | 81 + .../webapps/recon/ozone-recon-web/src/App.tsx | 2 +- .../src/components/OverviewCard/OverviewCard.less | 13 +- .../src/components/OverviewCard/OverviewCard.tsx | 17 +- .../StorageBar.less} | 41 +- .../src/components/StorageBar/StorageBar.tsx | 70 + .../ozone-recon-web/src/makeRouteWithSubRoutes.tsx | 4 +- .../webapps/recon/ozone-recon-web/src/routes.tsx | 2 +- .../{routes.types.tsx => types/datanode.types.tsx} | 10 +- .../src/{ => types}/routes.types.tsx | 0 .../recon/ozone-recon-web/src/utils/themeIcons.tsx | 89 + .../src/views/Datanodes/Datanodes.less | 16 +- .../src/views/Datanodes/Datanodes.tsx | 82 +- .../src/views/Overview/Overview.tsx | 86 +- .../src/views/Pipelines/Pipelines.tsx | 49 +- .../webapps/recon/ozone-recon-web/yarn.lock | 13 +- .../ozone/recon/AbstractOMMetadataManagerTest.java | 15 +- ...rKeyService.java => TestContainerEndpoint.java} | 173 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 408 +++++ .../ozone/recon/api/TestTaskStatusService.java | 8 +- .../ozone/recon/api/TestUtilizationService.java | 77 - .../ozone/recon/fsck/TestMissingContainerTask.java | 126 ++ .../TestReconInternalSchemaDefinition.java | 12 +- .../scm/AbstractReconContainerManagerTest.java | 38 +- .../ozone/recon/scm/TestReconContainerManager.java | 16 + ...TestReconIncrementalContainerReportHandler.java | 36 +- ...rManagerTest.java => TestReconNodeManager.java} | 69 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 7 +- .../hadoop/ozone/recon/tasks/DummyReconDBTask.java | 2 +- .../ozone/recon/tasks/TestFileSizeCountTask.java | 125 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 108 +- .../recon/tasks/TestReconTaskControllerImpl.java | 55 +- .../recon/types/GuiceInjectorUtilsForTests.java | 27 +- ...uthParser.java => AWSV4SignatureProcessor.java} | 204 ++- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 1 + .../apache/hadoop/ozone/s3/HeaderPreprocessor.java | 16 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 96 +- .../hadoop/ozone/s3/RootPageDisplayFilter.java | 61 + .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 4 +- .../hadoop/ozone/s3/S3GatewayHttpServer.java | 2 +- ...{AWSAuthParser.java => SignatureProcessor.java} | 28 +- .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 10 - .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 5 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 13 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 17 +- .../s3/header/AuthenticationHeaderParser.java | 69 - .../ozone/s3/header/AuthorizationHeaderV4.java | 30 +- .../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 4 +- .../ozone/s3/util/RangeHeaderParserUtil.java | 2 +- .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 2 +- .../resources/webapps/s3gateway/WEB-INF/web.xml | 10 +- .../ozone/s3/TestAWSV4SignatureProcessor.java | 103 ++ .../hadoop/ozone/s3/TestOzoneClientProducer.java | 13 +- .../ozone/s3/TestVirtualHostStyleFilter.java | 17 +- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 30 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 27 +- .../ozone/s3/header/TestAuthorizationHeaderV4.java | 2 +- .../hadoop/ozone/freon/BaseAppendLogGenerator.java | 60 + .../hadoop/ozone/freon/BaseFreonGenerator.java | 133 +- .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 188 +++ ...hunkGenerator.java => DatanodeBlockPutter.java} | 143 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 2 +- .../freon/FollowerAppendLogEntryGenerator.java | 468 ++++++ .../java/org/apache/hadoop/ozone/freon/Freon.java | 14 +- .../apache/hadoop/ozone/freon/FreonHttpServer.java | 2 +- .../hadoop/ozone/freon/OmBucketGenerator.java | 23 +- .../apache/hadoop/ozone/freon/OmKeyGenerator.java | 24 +- .../ozone/freon/OzoneClientKeyGenerator.java | 21 +- .../ozone/freon/OzoneClientKeyValidator.java | 9 +- .../org/apache/hadoop/ozone/freon/ProgressBar.java | 10 +- .../hadoop/ozone/freon/RandomKeyGenerator.java | 12 +- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 6 +- .../ozone/genesis/BenchmarkChunkManager.java | 181 +++ .../org/apache/hadoop/ozone/genesis/Genesis.java | 11 + .../hadoop/ozone/segmentparser/BaseLogParser.java | 55 + .../segmentparser/DatanodeRatisLogParser.java | 60 + .../segmentparser/GenericRatisLogParser.java} | 33 +- .../ozone/segmentparser/OMRatisLogParser.java | 46 +- .../hadoop/ozone/segmentparser/RatisLogParser.java | 50 +- .../hadoop/ozone/segmentparser/package-info.java} | 9 +- pom.xml | 207 +-- 837 files changed, 21438 insertions(+), 7666 deletions(-) diff --cc hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot index 7275a76,89472f2..adc6e0f --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot @@@ -19,53 -19,34 +19,55 @@@ Library OperatingSyste Resource ../commonlib.robot *** Variables *** +${OfsBucket1} om:9862/fstest/bucket1 +${O3fsBucket1} bucket1.fstest/ +${O3Bucket1} o3://om/fstest/bucket1 +${O3Bucket2} o3://om/fstest/bucket2 +${O3Bucket3} o3://om/fstest2/bucket3 +${OfsNonExistBucket} om:9862/abc/def +${O3fsNonExistBucket} def.abc/ +${NonExistVolume} abc *** Test Cases *** -Create volume and bucket +Create volume and bucket for Ozone file System test Execute ozone sh volume create o3://om/fstest --quota 100TB + Execute ozone sh bucket create ${O3Bucket1} + Execute ozone sh bucket create ${O3Bucket2} + Execute ozone sh volume create o3://om/fstest2 --quota 100TB - Execute ozone sh bucket create o3://om/fstest/bucket1 - Execute ozone sh bucket create o3://om/fstest/bucket2 - Execute ozone sh bucket create o3://om/fstest2/bucket3 + Execute ozone sh bucket create ${O3Bucket3} + + +Check volume from o3fs + ${result} = Execute ozone fs -ls o3fs://${O3fsBucket1} -Check volume from ozonefs - ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/ +Test ozone shell with ofs + Test ozone shell with scheme ofs om:9862/fstest/bucket1 om:9862/fstest/bucket2 om:9862/fstest2/bucket3 om:9862/abc/def -Run ozoneFS tests - Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep - ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name' +Test ozone shell with o3fs + Test ozone shell with scheme o3fs bucket1.fstest/ bucket2.fstest/ bucket3.fstest2/ def.abc/ + +*** Keywords *** +Test ozone shell with scheme + [arguments] ${scheme} ${testBucket1} ${testBucket2} ${testBucket3} ${nonExistBucket} + + ${result} = Execute ozone fs -ls ${scheme}://${testBucket1} + Execute ozone fs -mkdir -p ${scheme}://${testBucket1}/testdir/deep + ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name' Should contain ${result} testdir/deep - Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/ - ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name' + + Execute ozone fs -copyFromLocal NOTICE.txt ${scheme}://${testBucket1}/testdir/deep/ + ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name' Should contain ${result} NOTICE.txt + ${result} = Execute ozone sh key info o3://om/fstest/bucket1/testdir/deep/NOTICE.txt | jq -r '.replicationFactor' + Should Be Equal ${result} 3 - Execute ozone fs -put NOTICE.txt o3fs://bucket1.fstest/testdir/deep/PUTFILE.txt - ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name' + Execute ozone fs -put NOTICE.txt ${scheme}://${testBucket1}/testdir/deep/PUTFILE.txt + ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name' Should contain ${result} PUTFILE.txt - ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/testdir/deep/ + ${result} = Execute ozone fs -ls ${scheme}://${testBucket1}/testdir/deep/ Should contain ${result} NOTICE.txt Should contain ${result} PUTFILE.txt --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
