This is an automated email from the ASF dual-hosted git repository. nanda pushed a commit to branch HDDS-2823 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6de98c6d124ec127310e1aa9fac0cbc576991cfc Merge: 40127b3 10df489 Author: Nandakumar <[email protected]> AuthorDate: Sat Oct 24 20:07:04 2020 +0530 Merge branch 'master' into HDDS-2823 .github/close-pending.sh | 41 + .github/closing-message.txt | 7 + .github/comment-commands/close.sh | 10 +- .github/comment-commands/pending.sh | 1 + .github/workflows/close-pending.yaml | 32 + .github/workflows/comments.yaml | 2 +- .github/workflows/post-commit.yml | 411 ++-- LICENSE.txt | 4 +- dev-support/byteman/appendlog.btm | 28 + dev-support/byteman/hcfs-read.btm | 67 + dev-support/byteman/hcfs-write.btm | 111 ++ dev-support/byteman/ratis-flush.btm | 34 + dev-support/byteman/ratis-no-flush.btm | 25 + dev-support/byteman/watchforcommit.btm | 35 + dev-support/byteman/watchforcommit_all.btm | 47 + hadoop-hdds/client/pom.xml | 15 +- .../hadoop/hdds/scm/XceiverClientFactory.java | 38 + .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 48 +- .../hadoop/hdds/scm/XceiverClientManager.java | 40 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 11 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 8 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 62 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 158 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 49 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 10 +- .../hadoop/hdds/scm/storage/CommitWatcher.java | 37 +- .../hdds/scm/storage/DummyChunkInputStream.java | 2 +- .../storage/TestBlockOutputStreamCorrectness.java | 224 +++ .../hadoop/hdds/scm/storage/TestBufferPool.java | 46 + hadoop-hdds/common/pom.xml | 14 +- hadoop-hdds/common/src/main/conf/hadoop-env.sh | 13 +- .../org/apache/hadoop/hdds/cli/GenericCli.java | 21 + .../hadoop/hdds/cli/SubcommandWithParent.java | 30 + .../org/apache/hadoop/hdds/cli/package-info.java | 4 +- .../org/apache/hadoop/hdds/client/OzoneQuota.java | 240 ++- .../hadoop/hdds/protocol/DatanodeDetails.java | 159 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 160 +- .../hadoop/hdds/ratis/conf/RatisClientConfig.java | 110 +- .../RequestTypeDependentRetryPolicyCreator.java | 120 ++ .../retrypolicy/RetryLimitedPolicyCreator.java | 47 + .../hdds/ratis/retrypolicy/RetryPolicyCreator.java | 29 + .../hdds/ratis/retrypolicy/package-info.java | 23 + .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 4 + .../hadoop/hdds/scm/ByteStringConversion.java | 18 +- .../hadoop/hdds/scm/PipelineChoosePolicy.java | 37 + .../hdds/scm/PipelineRequestInformation.java | 59 + .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 23 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 13 + .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 15 +- .../hadoop/hdds/scm/container/ContainerInfo.java | 2 + .../hadoop/hdds/scm/exceptions/SCMException.java | 4 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 2 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 42 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 229 ++- .../x509/certificate/utils/CertificateCodec.java | 2 +- .../hadoop/hdds/utils/BackgroundService.java | 73 +- .../apache/hadoop/hdds/utils/BackgroundTask.java | 4 +- .../hadoop/hdds/utils/BackgroundTaskQueue.java | 5 +- .../apache/hadoop/hdds/utils/RatisVersionInfo.java | 61 + .../org/apache/hadoop/hdds/utils/Scheduler.java | 2 +- .../org/apache/hadoop/hdds/utils/VersionInfo.java | 9 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 4 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 56 +- .../apache/hadoop/ozone/common/ChunkBuffer.java | 14 +- .../common/ChunkBufferImplWithByteBuffer.java | 10 +- .../org/apache/hadoop/ozone/common/Storage.java | 2 +- .../container/common/helpers/ChunkInfoList.java | 56 + .../common/src/main/resources/ozone-default.xml | 82 +- .../hadoop/hdds/conf/SimpleConfiguration.java | 13 + .../hadoop/hdds/conf/TestOzoneConfiguration.java | 7 +- .../hadoop/hdds/protocol/MockDatanodeDetails.java | 2 +- .../hdds/ratis/conf/TestRaftClientConfig.java | 62 + .../hdds/ratis/conf/TestRatisClientConfig.java | 68 + .../hadoop/hdds/scm/pipeline/MockPipeline.java | 29 +- .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 62 +- .../hadoop/ozone/common/TestChunkBuffer.java | 16 +- hadoop-hdds/config/pom.xml | 4 +- .../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +- .../hdds/conf/ConfigurationReflectionUtil.java | 57 + .../hadoop/hdds/conf/InMemoryConfiguration.java | 58 + .../hdds/conf/TestConfigurationReflectionUtil.java | 111 ++ hadoop-hdds/container-service/pom.xml | 11 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 18 +- .../container/common/helpers/ContainerUtils.java | 66 +- .../container/common/impl/ContainerDataYaml.java | 3 + .../container/common/impl/HddsDispatcher.java | 61 +- .../common/impl/StorageLocationReport.java | 11 +- .../container/common/interfaces/BlockIterator.java | 5 +- .../container/common/interfaces/Container.java | 7 - .../common/statemachine/DatanodeStateMachine.java | 51 +- .../common/statemachine/SCMConnectionManager.java | 9 +- .../common/statemachine/StateContext.java | 31 + .../CloseContainerCommandHandler.java | 2 +- .../CreatePipelineCommandHandler.java | 15 +- .../commandhandler/DeleteBlocksCommandHandler.java | 92 +- .../states/datanode/RunningDatanodeState.java | 14 +- .../states/endpoint/RegisterEndpointTask.java | 6 +- .../common/transport/server/XceiverServerSpi.java | 22 +- .../server/ratis/ContainerStateMachine.java | 14 +- .../transport/server/ratis/XceiverServerRatis.java | 93 +- .../container/common/utils/ContainerCache.java | 95 +- .../container/common/utils/HddsVolumeUtil.java | 7 +- .../container/common/utils/ReferenceCountedDB.java | 10 +- .../container/common/volume/MutableVolumeSet.java | 10 +- .../container/keyvalue/KeyValueBlockIterator.java | 156 -- .../container/keyvalue/KeyValueContainer.java | 29 +- .../container/keyvalue/KeyValueContainerCheck.java | 12 +- .../container/keyvalue/KeyValueContainerData.java | 48 +- .../ozone/container/keyvalue/KeyValueHandler.java | 18 +- .../container/keyvalue/helpers/BlockUtils.java | 2 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 184 +- .../container/keyvalue/impl/BlockManagerImpl.java | 124 +- .../keyvalue/interfaces/BlockManager.java | 12 + .../background/BlockDeletingService.java | 75 +- .../metadata/AbstractDatanodeDBDefinition.java | 74 + .../container/metadata/AbstractDatanodeStore.java | 297 +++ .../ozone/container/metadata/BlockDataCodec.java | 47 + .../container/metadata/ChunkInfoListCodec.java | 45 + .../metadata/DatanodeSchemaOneDBDefinition.java | 91 + .../metadata/DatanodeSchemaTwoDBDefinition.java | 81 + .../ozone/container/metadata/DatanodeStore.java | 94 + .../metadata/DatanodeStoreSchemaOneImpl.java | 49 + .../metadata/DatanodeStoreSchemaTwoImpl.java | 44 + .../ozone/container/metadata/DatanodeTable.java | 130 ++ .../metadata/SchemaOneChunkInfoListCodec.java | 68 + .../metadata/SchemaOneDeletedBlocksTable.java | 180 ++ .../container/metadata/SchemaOneKeyCodec.java | 106 ++ .../ozone/container/metadata/package-info.java | 22 + .../ozone/container/ozoneimpl/ContainerReader.java | 2 + .../ozone/container/ozoneimpl/OzoneContainer.java | 30 +- .../replication/GrpcReplicationClient.java | 29 +- .../replication/SimpleContainerDownloader.java | 30 +- .../protocol/StorageContainerDatanodeProtocol.java | 14 +- .../protocol/commands/CreatePipelineCommand.java | 46 +- ...inerDatanodeProtocolClientSideTranslatorPB.java | 12 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 4 +- .../main/resources/webapps/hddsDatanode/index.html | 4 +- .../ozone/container/ContainerTestHelper.java | 5 + .../hadoop/ozone/container/common/ScmTestMock.java | 9 +- .../container/common/TestBlockDeletingService.java | 113 +- .../ozone/container/common/TestContainerCache.java | 73 +- .../common/TestKeyValueContainerData.java | 3 + .../TestSchemaOneBackwardsCompatibility.java | 626 +++++++ .../common/impl/TestContainerDataYaml.java | 3 + .../common/statemachine/TestStateContext.java | 85 + .../TestCreatePipelineCommandHandler.java | 7 +- .../states/datanode/TestRunningDatanodeState.java | 90 + .../keyvalue/TestKeyValueBlockIterator.java | 340 ++-- .../container/keyvalue/TestKeyValueContainer.java | 75 +- .../keyvalue/TestKeyValueContainerCheck.java | 18 +- .../container/ozoneimpl/TestContainerReader.java | 128 +- .../container/ozoneimpl/TestOzoneContainer.java | 65 +- .../test/resources/123-dn-container.db/000024.sst | Bin 0 -> 1022 bytes .../test/resources/123-dn-container.db/000026.sst | Bin 0 -> 827 bytes .../test/resources/123-dn-container.db/000032.sst | Bin 0 -> 896 bytes .../test/resources/123-dn-container.db/000034.log | 0 .../src/test/resources/123-dn-container.db/CURRENT | 1 + .../test/resources/123-dn-container.db/IDENTITY | 1 + .../resources/123-dn-container.db/MANIFEST-000033 | Bin 0 -> 297 bytes .../resources/123-dn-container.db/OPTIONS-000033 | 165 ++ .../resources/123-dn-container.db/OPTIONS-000036 | 165 ++ .../src/test/resources/123.container | 10 + hadoop-hdds/docs/content/_index.md | 5 +- hadoop-hdds/docs/content/_index.zh.md | 2 +- hadoop-hdds/docs/content/beyond/Containers.md | 234 --- hadoop-hdds/docs/content/beyond/Containers.zh.md | 203 -- .../docs/content/beyond/DockerCheatSheet.md | 88 - .../docs/content/beyond/DockerCheatSheet.zh.md | 85 - hadoop-hdds/docs/content/beyond/_index.md | 30 - hadoop-hdds/docs/content/beyond/_index.zh.md | 27 - hadoop-hdds/docs/content/concept/Containers.md | 47 + hadoop-hdds/docs/content/concept/Containers.png | Bin 0 -> 24775 bytes hadoop-hdds/docs/content/concept/Datanodes.md | 5 +- hadoop-hdds/docs/content/concept/Datanodes.zh.md | 3 + hadoop-hdds/docs/content/concept/Hdds.md | 52 - hadoop-hdds/docs/content/concept/Hdds.zh.md | 40 - hadoop-hdds/docs/content/concept/Overview.md | 7 +- hadoop-hdds/docs/content/concept/Overview.zh.md | 7 +- .../docs/content/concept/OzoneManager-ReadPath.png | Bin 0 -> 81030 bytes .../content/concept/OzoneManager-WritePath.png | Bin 0 -> 96696 bytes hadoop-hdds/docs/content/concept/OzoneManager.md | 63 +- hadoop-hdds/docs/content/concept/OzoneManager.png | Bin 0 -> 13327 bytes .../docs/content/concept/OzoneManager.zh.md | 9 + .../content/concept/StorageContainerManager.md | 99 + .../content/concept/StorageContainerManager.png | Bin 0 -> 13336 bytes .../content/concept/StorageContainerManager.zh.md | 49 + hadoop-hdds/docs/content/concept/_index.md | 4 +- hadoop-hdds/docs/content/concept/_index.zh.md | 2 +- hadoop-hdds/docs/content/design/ec.md | 39 + hadoop-hdds/docs/content/design/multiraft.md | 2 +- .../docs/content/design/namespace-support.md | 6 +- hadoop-hdds/docs/content/design/ofs.md | 131 -- .../content/design/ozone-enhancement-proposals.md | 2 +- hadoop-hdds/docs/content/design/recon2.md | 2 +- hadoop-hdds/docs/content/design/scmha.md | 4 +- hadoop-hdds/docs/content/design/storage-class.md | 28 + hadoop-hdds/docs/content/design/topology.md | 29 + hadoop-hdds/docs/content/design/trash.md | 7 +- hadoop-hdds/docs/content/design/typesafeconfig.md | 10 +- ...e-volume-management.md => volume-management.md} | 0 hadoop-hdds/docs/content/feature/GDPR.md | 80 + hadoop-hdds/docs/content/feature/GDPR.zh.md | 41 + .../docs/content/feature/HA-OM-doublebuffer.png | Bin 0 -> 77661 bytes hadoop-hdds/docs/content/feature/HA-OM.png | Bin 0 -> 60888 bytes hadoop-hdds/docs/content/feature/HA.md | 115 ++ hadoop-hdds/docs/content/feature/Observability.md | 224 +++ hadoop-hdds/docs/content/feature/Recon.md | 47 + hadoop-hdds/docs/content/feature/Topology.md | 108 ++ hadoop-hdds/docs/content/feature/_index.md | 34 + .../docs/content/{gdpr => feature}/_index.zh.md | 0 hadoop-hdds/docs/content/gdpr/GDPR in Ozone.md | 42 - hadoop-hdds/docs/content/gdpr/GDPR in Ozone.zh.md | 36 - hadoop-hdds/docs/content/gdpr/_index.md | 38 - hadoop-hdds/docs/content/interface/CSI.md | 22 +- hadoop-hdds/docs/content/interface/CSI.png | Bin 0 -> 27210 bytes hadoop-hdds/docs/content/interface/CSI.zh.md | 92 + hadoop-hdds/docs/content/interface/Cli.md | 208 +++ hadoop-hdds/docs/content/interface/JavaApi.md | 5 +- hadoop-hdds/docs/content/interface/JavaApi.zh.md | 3 + hadoop-hdds/docs/content/interface/O3fs.md | 127 ++ hadoop-hdds/docs/content/interface/O3fs.zh.md | 167 ++ hadoop-hdds/docs/content/interface/Ofs.md | 227 +++ hadoop-hdds/docs/content/interface/OzoneFS.md | 167 -- hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 159 -- hadoop-hdds/docs/content/interface/S3.md | 29 +- hadoop-hdds/docs/content/interface/S3.zh.md | 3 + hadoop-hdds/docs/content/interface/_index.md | 4 +- hadoop-hdds/docs/content/recipe/Prometheus.md | 5 +- hadoop-hdds/docs/content/recipe/Prometheus.zh.md | 7 +- hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 1 - .../docs/content/security/SecuityWithRanger.md | 43 - .../docs/content/security/SecuityWithRanger.zh.md | 35 - hadoop-hdds/docs/content/security/SecureOzone.md | 3 + .../docs/content/security/SecureOzone.zh.md | 6 +- .../docs/content/security/SecuringDatanodes.md | 39 +- .../docs/content/security/SecuringDatanodes.zh.md | 53 + .../docs/content/security/SecuringOzoneHTTP.md | 7 +- hadoop-hdds/docs/content/security/SecuringS3.md | 5 +- hadoop-hdds/docs/content/security/SecuringS3.zh.md | 3 + hadoop-hdds/docs/content/security/SecuringTDE.md | 9 +- .../docs/content/security/SecuringTDE.zh.md | 55 + hadoop-hdds/docs/content/security/SecurityAcls.md | 11 +- .../docs/content/security/SecurityAcls.zh.md | 69 + .../docs/content/security/SecurityWithRanger.md | 46 + .../docs/content/security/SecurityWithRanger.zh.md | 38 + hadoop-hdds/docs/content/shell/BucketCommands.md | 100 - .../docs/content/shell/BucketCommands.zh.md | 98 - hadoop-hdds/docs/content/shell/Format.md | 69 - hadoop-hdds/docs/content/shell/Format.zh.md | 65 - hadoop-hdds/docs/content/shell/KeyCommands.md | 177 -- hadoop-hdds/docs/content/shell/KeyCommands.zh.md | 176 -- hadoop-hdds/docs/content/shell/VolumeCommands.md | 114 -- .../docs/content/shell/VolumeCommands.zh.md | 108 -- hadoop-hdds/docs/content/shell/_index.md | 28 - hadoop-hdds/docs/content/shell/_index.zh.md | 27 - hadoop-hdds/docs/content/start/FromSource.md | 38 +- hadoop-hdds/docs/content/start/FromSource.zh.md | 7 +- hadoop-hdds/docs/content/start/OnPrem.md | 2 +- hadoop-hdds/docs/content/start/OnPrem.zh.md | 2 +- .../docs/content/start/StartFromDockerHub.md | 6 +- hadoop-hdds/docs/content/tools/TestTools.md | 14 +- hadoop-hdds/docs/content/tools/TestTools.zh.md | 14 +- hadoop-hdds/docs/content/tools/_index.md | 6 +- hadoop-hdds/docs/pom.xml | 4 +- .../themes/ozonedoc/layouts/_default/single.html | 2 +- .../themes/ozonedoc/layouts/design/section.html | 2 +- .../ozonedoc/layouts/partials/languages.html | 3 +- .../themes/ozonedoc/layouts/partials/navbar.html | 6 +- .../themes/ozonedoc/layouts/partials/sidebar.html | 14 +- .../docs/themes/ozonedoc/static/css/ozonedoc.css | 23 +- hadoop-hdds/framework/pom.xml | 4 +- .../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 7 +- .../hdds/conf/DatanodeRatisServerConfig.java | 32 +- .../x509/certificate/authority/BaseApprover.java | 11 +- .../certificate/authority/DefaultApprover.java | 11 + .../certificate/authority/DefaultCAServer.java | 42 +- .../certificate/client/DNCertificateClient.java | 5 +- .../certificates/utils/CertificateSignRequest.java | 2 +- .../certificates/utils/SelfSignedCertificate.java | 109 +- .../server/OzoneProtocolMessageDispatcher.java | 41 +- .../hadoop/hdds/server/http/HttpServer2.java | 21 + .../apache/hadoop/hdds/utils/HddsServerUtil.java | 29 +- .../hadoop/hdds/utils/MetadataKeyFilters.java | 42 +- .../apache/hadoop/hdds/utils/db/DBDefinition.java | 41 +- .../org/apache/hadoop/hdds/utils/db/DBStore.java | 8 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 41 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 17 +- .../hadoop/hdds/utils/db/RDBStoreIterator.java | 40 +- .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 105 ++ .../org/apache/hadoop/hdds/utils/db/Table.java | 69 + .../apache/hadoop/hdds/utils/db/TypedTable.java | 59 + .../resources/webapps/static/angular-1.7.9.min.js | 350 ---- .../resources/webapps/static/angular-1.8.0.min.js | 350 ++++ .../webapps/static/angular-route-1.7.9.min.js | 17 - .../webapps/static/angular-route-1.8.0.min.js | 17 + .../x509/certificates/TestRootCertificate.java | 42 +- .../apache/hadoop/hdds/server/TestJsonUtils.java | 5 +- .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 10 +- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 61 + hadoop-hdds/hadoop-dependency-client/README.md | 18 +- hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 8 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +- hadoop-hdds/interface-admin/pom.xml | 4 +- .../src/main/{proto => resources}/proto.lock | 0 hadoop-hdds/interface-client/pom.xml | 4 +- .../src/main/proto/DatanodeClientProtocol.proto | 4 + .../interface-client/src/main/proto/hdds.proto | 16 +- .../interface-client/src/main/proto/proto.lock | 1938 ------------------- .../interface-client/src/main/resources/proto.lock | 1958 ++++++++++++++++++++ hadoop-hdds/interface-server/pom.xml | 4 +- .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 9 +- .../src/main/proto/ScmServerProtocol.proto | 2 + .../interface-server/src/main/proto/proto.lock | 1786 ------------------ .../interface-server/src/main/resources/proto.lock | 1786 ++++++++++++++++++ hadoop-hdds/pom.xml | 20 +- hadoop-hdds/server-scm/pom.xml | 5 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 14 +- .../block/DatanodeDeletedBlockTransactions.java | 32 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 24 +- .../hdds/scm/block/SCMBlockDeletingService.java | 7 +- .../container/AbstractContainerReportHandler.java | 101 +- .../hdds/scm/container/ContainerReplica.java | 43 +- .../hdds/scm/container/ContainerReportHandler.java | 14 +- .../hdds/scm/container/ContainerStateManager.java | 8 +- .../IncrementalContainerReportHandler.java | 2 +- .../hdds/scm/container/ReplicationManager.java | 133 +- .../hdds/scm/container/SCMContainerManager.java | 70 +- .../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 55 + .../hadoop/hdds/scm/node/DeadNodeHandler.java | 2 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 6 + .../hadoop/hdds/scm/node/NodeStateManager.java | 30 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 68 + .../scm/pipeline/BackgroundPipelineCreator.java | 35 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 4 + .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 42 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 2 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 39 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 40 +- .../algorithms/HealthyPipelineChoosePolicy.java | 46 + .../algorithms/PipelineChoosePolicyFactory.java | 106 ++ .../algorithms/RandomPipelineChoosePolicy.java | 38 + .../pipeline/choose/algorithms/package-info.java | 18 + .../algorithms/DefaultLeaderChoosePolicy.java | 42 + .../choose/algorithms/LeaderChoosePolicy.java | 55 + .../algorithms/LeaderChoosePolicyFactory.java | 75 + .../algorithms/MinLeaderCountChoosePolicy.java | 91 + .../leader/choose/algorithms/package-info.java | 19 + .../SCMSecurityProtocolServerSideTranslatorPB.java | 17 +- ...lockLocationProtocolServerSideTranslatorPB.java | 2 +- ...inerLocationProtocolServerSideTranslatorPB.java | 17 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 7 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 4 +- .../apache/hadoop/hdds/scm/server/SCMMXBean.java | 6 + .../hdds/scm/server/StorageContainerManager.java | 31 +- .../src/main/resources/webapps/scm/index.html | 4 +- .../main/resources/webapps/scm/scm-overview.html | 27 +- .../src/main/resources/webapps/scm/scm.js | 4 - .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 47 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 135 ++ .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 17 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 50 + .../scm/container/TestContainerReportHandler.java | 197 +- .../hdds/scm/container/TestReplicationManager.java | 92 + .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 5 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 85 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 161 ++ ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 180 ++ .../TestPipelineChoosePolicyFactory.java | 94 + .../choose/algorithms/TestLeaderChoosePolicy.java | 74 + .../ozone/container/common/TestEndPoint.java | 2 +- .../testutils/ReplicationNodeManagerMock.java | 15 + hadoop-hdds/test-utils/pom.xml | 4 +- hadoop-hdds/tools/pom.xml | 12 +- .../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 67 + .../org/apache/hadoop/hdds/cli/package-info.java | 22 + .../hdds/scm/cli/ReplicationManagerCommands.java | 23 +- .../scm/cli/ReplicationManagerStartSubcommand.java | 21 +- .../cli/ReplicationManagerStatusSubcommand.java | 32 +- .../scm/cli/ReplicationManagerStopSubcommand.java | 25 +- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 40 +- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 27 +- .../hdds/scm/cli/SafeModeExitSubcommand.java | 22 +- .../hdds/scm/cli/SafeModeWaitSubcommand.java | 13 +- .../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 72 + .../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 43 + .../hadoop/hdds/scm/cli/TopologySubcommand.java | 65 +- .../hdds/scm/cli/container/CloseSubcommand.java | 20 +- .../hdds/scm/cli/container/ContainerCommands.java | 21 +- .../hdds/scm/cli/container/CreateSubcommand.java | 26 +- .../hdds/scm/cli/container/DeleteSubcommand.java | 20 +- .../hdds/scm/cli/container/InfoSubcommand.java | 48 +- .../hdds/scm/cli/container/ListSubcommand.java | 32 +- .../hdds/scm/cli/container/WithScmClient.java | 29 - .../hdds/scm/cli/datanode/DatanodeCommands.java | 21 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 48 +- .../cli/pipeline/ActivatePipelineSubcommand.java | 19 +- .../scm/cli/pipeline/ClosePipelineSubcommand.java | 19 +- .../scm/cli/pipeline/CreatePipelineSubcommand.java | 48 +- .../cli/pipeline/DeactivatePipelineSubcommand.java | 19 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 40 +- .../hdds/scm/cli/pipeline/PipelineCommands.java | 22 +- hadoop-ozone/client/pom.xml | 4 +- .../org/apache/hadoop/ozone/client/BucketArgs.java | 72 +- .../apache/hadoop/ozone/client/ObjectStore.java | 3 + .../apache/hadoop/ozone/client/OzoneBucket.java | 126 +- .../hadoop/ozone/client/OzoneClientFactory.java | 8 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 103 +- .../org/apache/hadoop/ozone/client/VolumeArgs.java | 45 +- .../ozone/client/io/BlockOutputStreamEntry.java | 30 +- .../client/io/BlockOutputStreamEntryPool.java | 40 +- .../hadoop/ozone/client/io/KeyInputStream.java | 98 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 60 +- .../hadoop/ozone/client/io/OzoneInputStream.java | 5 + .../ozone/client/protocol/ClientProtocol.java | 29 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 117 +- hadoop-ozone/common/pom.xml | 4 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 75 + .../org/apache/hadoop/ozone/audit/OMAction.java | 6 +- .../apache/hadoop/ozone/freon/OzoneGetConf.java | 278 --- .../apache/hadoop/ozone/freon/package-info.java | 21 - .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 + .../hadoop/ozone/om/exceptions/OMException.java | 11 +- .../ozone/om/exceptions/OMNotLeaderException.java | 2 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 253 ++- .../ozone/om/helpers/BucketEncryptionKeyInfo.java | 4 + .../hadoop/ozone/om/helpers/OmBucketArgs.java | 50 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 214 ++- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 18 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +- .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 24 +- .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 111 +- .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 215 --- .../hadoop/ozone/om/helpers/OmRenameKeys.java | 59 + .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 64 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 30 + .../hadoop/ozone/om/helpers/OzoneFileStatus.java | 18 + .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 9 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 15 +- .../ozone/om/protocolPB/Hadoop3OmTransport.java | 175 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 44 +- .../ozone/security/OzoneTokenIdentifier.java | 61 +- .../org/apache/hadoop/ozone/util/ExitManager.java | 33 + .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 15 +- .../java/org/apache/hadoop/ozone/TestOmUtils.java | 49 + .../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 22 +- .../om/helpers/TestOmKeyLocationInfoGroup.java | 59 + .../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 4 +- .../hadoop/ozone/om/helpers/TestOzoneFsUtils.java | 39 + hadoop-ozone/csi/pom.xml | 4 +- .../csi/src/main/{proto => resources}/proto.lock | 0 hadoop-ozone/datanode/pom.xml | 4 +- hadoop-ozone/dev-support/checks/acceptance.sh | 2 + hadoop-ozone/dev-support/checks/bats.sh | 35 + hadoop-ozone/dev-support/checks/build.sh | 2 +- hadoop-ozone/dev-support/checks/findbugs.sh | 7 +- hadoop-ozone/dev-support/checks/kubernetes.sh | 36 + hadoop-ozone/dev-support/intellij/core-site.xml | 27 + hadoop-ozone/dist/README.md | 52 +- .../dist/dev-support/bin/dist-layout-stitching | 5 + hadoop-ozone/dist/pom.xml | 13 +- hadoop-ozone/dist/src/main/compose/failing1/.env | 1 + .../src/main/compose/failing1/docker-compose.yaml | 1 + .../dist/src/main/compose/failing1/docker-config | 1 + .../dist/src/main/compose/failing1/test.sh | 36 + hadoop-ozone/dist/src/main/compose/failing2/.env | 1 + .../src/main/compose/failing2/docker-compose.yaml | 1 + .../dist/src/main/compose/failing2/docker-config | 1 + .../dist/src/main/compose/failing2/test.sh | 36 + .../dist/src/main/compose/ozone-csi/docker-config | 3 + hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 19 + .../src/main/compose/ozone-ha/docker-compose.yaml | 93 + .../dist/src/main/compose/ozone-ha/docker-config | 35 + .../dist/src/main/compose/ozone-ha/test.sh | 33 + .../dist/src/main/compose/ozone-mr/common-config | 3 +- .../main/compose/ozone-mr/hadoop27/docker-config | 1 + .../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +- .../main/compose/ozone-mr/hadoop31/docker-config | 1 + .../src/main/compose/ozone-mr/hadoop31/test.sh | 7 +- .../main/compose/ozone-mr/hadoop32/docker-config | 1 + .../src/main/compose/ozone-mr/hadoop32/test.sh | 7 +- .../dist/src/main/compose/ozone-mr/test.sh | 37 + .../src/main/compose/ozone-om-ha-s3/docker-config | 5 +- .../dist/src/main/compose/ozone-om-ha-s3/test.sh | 2 + .../src/main/compose/ozone-om-ha/docker-config | 4 +- .../src/main/compose/ozone-topology/docker-config | 7 +- .../dist/src/main/compose/ozone-topology/test.sh | 4 +- .../dist/src/main/compose/ozone/docker-config | 7 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 20 +- .../src/main/compose/ozoneblockade/docker-config | 2 + .../src/main/compose/ozones3-haproxy/docker-config | 3 + .../src/main/compose/ozonesecure-mr/docker-config | 4 +- .../dist/src/main/compose/ozonesecure-mr/test.sh | 5 +- .../main/compose/ozonesecure-om-ha/docker-config | 11 +- .../src/main/compose/ozonesecure-om-ha/test.sh | 2 + .../src/main/compose/ozonesecure/docker-config | 9 +- .../dist/src/main/compose/ozonesecure/test.sh | 17 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 28 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 105 +- hadoop-ozone/dist/src/main/compose/upgrade/.env | 21 + .../dist/src/main/compose/upgrade/README.md | 29 + .../src/main/compose/upgrade/docker-compose.yaml | 127 ++ .../dist/src/main/compose/upgrade/docker-config | 34 + hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 69 + .../src/main/compose/upgrade/versions/README.md | 15 + .../main/compose/upgrade/versions/ozone-0.5.0.sh | 19 + .../main/compose/upgrade/versions/ozone-1.0.0.sh | 19 + hadoop-ozone/dist/src/main/k8s/.gitignore | 15 + .../src/main/k8s/definitions/ozone/config.yaml | 1 + .../k8s/definitions/ozone/definitions/onenode.yaml | 2 +- .../main/k8s/definitions/ozone/freon/freon.yaml | 2 +- .../{pv-test => test-webserver}/flekszible.yaml | 0 .../webserver-deployment.yaml | 0 .../webserver-service.yaml | 0 .../webserver-volume.yaml | 0 .../examples/getting-started/config-configmap.yaml | 1 + .../getting-started/datanode-statefulset.yaml | 10 - .../getting-started/freon/freon-deployment.yaml | 2 +- .../src/main/k8s/examples/getting-started/test.sh | 39 + .../k8s/examples/minikube/config-configmap.yaml | 1 + .../examples/minikube/freon/freon-deployment.yaml | 2 +- .../dist/src/main/k8s/examples/minikube/test.sh | 39 + .../src/main/k8s/examples/ozone-dev/Flekszible | 4 +- .../k8s/examples/ozone-dev/config-configmap.yaml | 1 + .../ozone-dev/csi/csi-provisioner-deployment.yaml | 2 +- .../examples/ozone-dev/datanode-statefulset.yaml | 10 - .../examples/ozone-dev/freon/freon-deployment.yaml | 2 +- .../dist/src/main/k8s/examples/ozone-dev/test.sh | 39 + .../dist/src/main/k8s/examples/ozone/Flekszible | 2 +- .../main/k8s/examples/ozone/config-configmap.yaml | 1 + .../ozone/csi/csi-provisioner-deployment.yaml | 2 +- .../k8s/examples/ozone/freon/freon-deployment.yaml | 2 +- .../dist/src/main/k8s/examples/ozone/test.sh | 39 + .../dist/src/main/k8s/examples/test-all.sh | 49 + hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 144 ++ hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +- .../dist/src/main/smoketest/admincli/admin.robot | 32 + .../src/main/smoketest/admincli/container.robot | 73 + .../src/main/smoketest/admincli/datanode.robot | 19 +- .../src/main/smoketest/admincli/pipeline.robot | 49 +- .../smoketest/admincli/replicationmanager.robot | 53 + .../src/main/smoketest/admincli/safemode.robot | 45 + .../main/smoketest/auditparser/auditparser.robot | 2 +- .../dist/src/main/smoketest/basic/basic.robot | 2 +- .../dist/src/main/smoketest/basic/getconf.robot | 46 + .../dist/src/main/smoketest/basic/links.robot | 152 ++ .../src/main/smoketest/basic/ozone-shell-lib.robot | 179 ++ .../main/smoketest/basic/ozone-shell-single.robot | 27 + .../src/main/smoketest/basic/ozone-shell.robot | 121 +- .../dist/src/main/smoketest/commonlib.robot | 34 +- .../dist/src/main/smoketest/createbucketenv.robot | 2 +- .../dist/src/main/smoketest/createmrenv.robot | 15 +- .../src/main/smoketest/debug/ozone-debug.robot | 6 +- .../dist/src/main/smoketest/failing/test1.robot | 21 + .../dist/src/main/smoketest/failing/test2.robot | 21 + .../dist/src/main/smoketest/freon/freon.robot | 2 +- .../dist/src/main/smoketest/gdpr/gdpr.robot | 2 +- hadoop-ozone/dist/src/main/smoketest/lib/os.robot | 49 + .../dist/src/main/smoketest/lib/os_tests.robot | 38 + .../dist/src/main/smoketest/mapreduce.robot | 11 +- .../main/smoketest/om-ratis/testOMAdminCmd.robot | 2 +- .../dist/src/main/smoketest/omha/testOMHA.robot | 2 +- .../dist/src/main/smoketest/ozone-lib/shell.robot | 53 + .../src/main/smoketest/ozone-lib/shell_tests.robot | 58 + .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 12 +- .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 2 +- .../dist/src/main/smoketest/ozonefs/setup.robot | 20 +- .../dist/src/main/smoketest/recon/recon-api.robot | 18 +- .../src/main/smoketest/s3/MultipartUpload.robot | 133 +- .../dist/src/main/smoketest/s3/bucketdelete.robot | 12 +- .../dist/src/main/smoketest/s3/buckethead.robot | 5 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 40 +- .../dist/src/main/smoketest/s3/objectcopy.robot | 23 +- .../dist/src/main/smoketest/s3/objectdelete.robot | 28 +- .../src/main/smoketest/s3/objectmultidelete.robot | 24 +- .../dist/src/main/smoketest/s3/objectputget.robot | 40 +- .../src/main/smoketest/s3/s3_compatbility_check.sh | 47 + .../smoketest/security/bucket-encryption.robot | 45 + .../main/smoketest/security/ozone-secure-s3.robot | 15 + .../smoketest/security/ozone-secure-token.robot | 16 +- .../dist/src/main/smoketest/spnego/web.robot | 16 +- .../src/main/smoketest/topology/loaddata.robot | 2 +- hadoop-ozone/dist/src/shell/hdds/hadoop-daemons.sh | 2 +- .../dist/src/shell/hdds/hadoop-functions.sh | 6 +- hadoop-ozone/dist/src/shell/ozone/ozone | 22 +- hadoop-ozone/dist/src/shell/ozone/stop-ozone.sh | 8 +- hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 23 + .../src/shell/upgrade/1.0.0/01-migrate-scm-db.sh | 24 + .../dist/src/test/shell/compose_testlib.bats | 37 + hadoop-ozone/dist/src/test/shell/gc_opts.bats | 6 +- hadoop-ozone/dist/src/test/shell/k8s_testlib.bats | 55 + hadoop-ozone/dist/src/test/shell/test1/test.sh | 15 + hadoop-ozone/dist/src/test/shell/test2/test.sh | 17 + .../dist/src/test/shell/test3/subtest1/test.sh | 17 + hadoop-ozone/dist/src/test/shell/test4/test.sh | 17 + .../fault-injection-test/mini-chaos-tests/pom.xml | 4 +- .../hadoop/ozone/TestMiniChaosOzoneCluster.java | 40 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + .../fault-injection-test/network-tests/pom.xml | 2 +- .../src/test/blockade/ozone/client.py | 10 +- hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 4 +- .../hadoop/ozone/insight/BaseInsightPoint.java | 34 +- .../ozone/insight/BaseInsightSubCommand.java | 6 + .../org/apache/hadoop/ozone/insight/Insight.java | 4 + .../apache/hadoop/ozone/insight/InsightPoint.java | 4 +- .../hadoop/ozone/insight/MetricGroupDisplay.java | 4 +- .../hadoop/ozone/insight/MetricsSubCommand.java | 33 +- .../datanode/DatanodeDispatcherInsight.java | 107 ++ .../insight/datanode/PipelineComponentUtil.java | 78 + .../ozone/insight/datanode/RatisInsight.java | 45 +- .../hadoop/ozone/insight/om/KeyManagerInsight.java | 2 +- .../hadoop/ozone/insight/om/OmProtocolInsight.java | 2 +- .../ozone/insight/scm/NodeManagerInsight.java | 2 +- .../ozone/insight/scm/ReplicaManagerInsight.java | 2 +- .../scm/ScmProtocolBlockLocationInsight.java | 2 +- .../scm/ScmProtocolContainerLocationInsight.java | 2 +- .../insight/scm/ScmProtocolDatanodeInsight.java | 2 +- .../insight/scm/ScmProtocolSecurityInsight.java | 2 +- hadoop-ozone/integration-test/pom.xml | 4 +- .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 391 ++++ .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 26 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 182 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 253 ++- .../hadoop/fs/ozone/contract/OzoneContract.java | 18 +- .../ozone/contract/rooted/RootedOzoneContract.java | 25 +- .../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 216 +++ .../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 2 +- .../TestRatisPipelineCreateAndDestroy.java | 2 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 20 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 13 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +- .../hadoop/ozone/TestStorageContainerManager.java | 35 +- .../ozone/TestStorageContainerManagerHelper.java | 31 +- .../ozone/client/rpc/Test2WayCommitInRatis.java | 16 +- .../rpc/TestBlockOutputStreamWithFailures.java | 32 +- ...estBlockOutputStreamWithFailuresFlushDelay.java | 32 +- .../rpc/TestCloseContainerHandlingByClient.java | 55 - .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 78 +- .../rpc/TestContainerReplicationEndToEnd.java | 19 +- .../TestContainerStateMachineFailureOnRead.java | 27 +- .../rpc/TestContainerStateMachineFailures.java | 47 +- .../client/rpc/TestDeleteWithSlowFollower.java | 28 +- .../client/rpc/TestDiscardPreallocatedBlocks.java | 186 ++ .../client/rpc/TestFailureHandlingByClient.java | 28 +- .../rpc/TestFailureHandlingByClientFlushDelay.java | 28 +- .../ozone/client/rpc/TestKeyInputStream.java | 119 +- .../rpc/TestMultiBlockWritesWithDnFailures.java | 28 +- .../rpc/TestOzoneClientRetriesOnException.java | 2 +- ...estOzoneClientRetriesOnExceptionFlushDelay.java | 2 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 845 ++++++++- .../hadoop/ozone/client/rpc/TestReadRetries.java | 40 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 3 - .../client/rpc/TestValidateBCSIDOnRestart.java | 36 +- .../ozone/client/rpc/TestWatchForCommit.java | 35 +- .../commandhandler/TestBlockDeletion.java | 171 +- .../ozoneimpl/TestOzoneContainerRatis.java | 138 -- .../container/ozoneimpl/TestRatisManager.java | 124 -- .../hadoop/ozone/freon/TestDataValidate.java | 18 +- .../ozone/freon/TestFreonWithDatanodeRestart.java | 17 +- .../ozone/freon/TestFreonWithPipelineDestroy.java | 17 +- .../ozone/freon/TestHadoopNestedDirGenerator.java | 203 ++ .../ozone/freon/TestOzoneClientKeyGenerator.java | 2 - .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 18 +- .../hadoop/ozone/fsck/TestContainerMapper.java | 3 + .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 32 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 215 ++- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 120 ++ .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 110 +- .../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 235 --- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 76 + .../ozone/om/TestOzoneManagerListVolumes.java | 18 +- .../hadoop/ozone/om/TestOzoneManagerRestart.java | 26 +- .../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 26 +- .../ozone/recon/TestReconWithOzoneManager.java | 46 +- .../hadoop/ozone/scm/TestCloseContainer.java | 148 ++ .../hadoop/ozone/scm/TestContainerSmallFile.java | 20 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 6 +- .../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 50 +- .../src/test/resources/log4j.properties | 3 - hadoop-ozone/interface-client/pom.xml | 4 +- .../src/main/proto/OmClientProtocol.proto | 60 + .../src/main/{proto => resources}/proto.lock | 0 .../dev-support/findbugsExcludeFile.xml | 21 + hadoop-ozone/interface-storage/pom.xml | 103 + .../apache/hadoop/ozone/om/OMMetadataManager.java | 385 ++++ .../ozone/om/codec/OMTransactionInfoCodec.java | 0 .../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 0 .../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 69 + .../ozone/om/codec/OmMultipartKeyInfoCodec.java | 0 .../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 59 + .../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 0 .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 68 + .../hadoop/ozone/om/codec/S3SecretValueCodec.java | 0 .../ozone/om/codec/TokenIdentifierCodec.java | 66 + .../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 0 .../apache/hadoop/ozone/om/codec/package-info.java | 24 + .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 216 +++ .../hadoop/ozone/om/helpers/OzoneAclStorage.java | 63 + .../ozone/om/helpers/OzoneAclStorageUtil.java | 62 + .../hadoop/ozone/om/helpers/package-info.java | 24 + .../org/apache/hadoop/ozone/om/package-info.java | 24 + .../hadoop/ozone/om/ratis/OMTransactionInfo.java | 163 ++ .../apache/hadoop/ozone/om/ratis/package-info.java | 24 + .../src/main/proto/OmStorageProtocol.proto | 60 + .../ozone/om/codec/TestOMTransactionInfoCodec.java | 0 .../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 116 ++ .../om/codec/TestOmMultipartKeyInfoCodec.java | 0 .../ozone/om/codec/TestOmPrefixInfoCodec.java | 0 .../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 121 ++ .../ozone/om/codec/TestS3SecretValueCodec.java | 0 .../apache/hadoop/ozone/om/codec/package-info.java | 0 .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 0 .../hadoop/ozone/om/helpers/package-info.java | 24 + hadoop-ozone/ozone-manager/pom.xml | 10 +- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 116 +- .../apache/hadoop/ozone/om/KeyDeletingService.java | 9 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 17 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 351 ++-- .../apache/hadoop/ozone/om/OMMetadataManager.java | 385 ---- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 37 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 70 +- .../hadoop/ozone/om/OpenKeyCleanupService.java | 45 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 744 ++++++-- .../org/apache/hadoop/ozone/om/ResolvedBucket.java | 111 ++ .../org/apache/hadoop/ozone/om/VolumeManager.java | 9 - .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 39 - .../hadoop/ozone/om/codec/OMDBDefinition.java | 19 +- .../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 58 - .../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 58 - .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 57 - .../ozone/om/codec/TokenIdentifierCodec.java | 56 - .../apache/hadoop/ozone/om/codec/package-info.java | 3 + .../apache/hadoop/ozone/om/fs/OzoneManagerFS.java | 53 +- .../hadoop/ozone/om/ratis/OMTransactionInfo.java | 153 -- .../ozone/om/ratis/OzoneManagerRatisServer.java | 18 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 26 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 40 +- .../hadoop/ozone/om/request/OMClientRequest.java | 94 +- .../om/request/bucket/OMBucketCreateRequest.java | 52 + .../request/bucket/OMBucketSetPropertyRequest.java | 64 + .../bucket/acl/OMBucketRemoveAclRequest.java | 2 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 2 +- .../om/request/file/OMDirectoryCreateRequest.java | 4 + .../ozone/om/request/file/OMFileCreateRequest.java | 57 +- .../ozone/om/request/file/OMFileRequest.java | 2 +- .../om/request/key/OMAllocateBlockRequest.java | 61 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 79 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 127 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 45 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 26 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 171 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 66 +- .../ozone/om/request/key/OMKeysRenameRequest.java | 271 +++ .../om/request/key/OMTrashRecoverRequest.java | 7 + .../om/request/key/acl/OMKeyRemoveAclRequest.java | 2 +- .../om/request/key/acl/OMKeySetAclRequest.java | 2 +- .../key/acl/prefix/OMPrefixRemoveAclRequest.java | 2 +- .../key/acl/prefix/OMPrefixSetAclRequest.java | 2 +- .../S3InitiateMultipartUploadRequest.java | 60 +- .../multipart/S3MultipartUploadAbortRequest.java | 71 +- .../S3MultipartUploadCommitPartRequest.java | 73 +- .../S3MultipartUploadCompleteRequest.java | 86 +- .../om/request/volume/OMVolumeCreateRequest.java | 6 + .../om/request/volume/OMVolumeSetQuotaRequest.java | 52 +- .../hadoop/ozone/om/response/CleanupTableInfo.java | 3 +- .../om/response/file/OMFileCreateResponse.java | 10 +- .../response/key/AbstractOMKeyDeleteResponse.java | 126 ++ .../om/response/key/OMAllocateBlockResponse.java | 18 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 18 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 20 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 73 +- .../om/response/key/OMKeysDeleteResponse.java | 56 +- .../om/response/key/OMKeysRenameResponse.java | 80 + .../om/response/key/OMOpenKeysDeleteRequest.java | 192 ++ .../om/response/key/OMOpenKeysDeleteResponse.java | 72 + .../multipart/S3MultipartUploadAbortResponse.java | 20 +- .../S3MultipartUploadCommitPartResponse.java | 21 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 10 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +- .../protocolPB/OzoneManagerRequestHandler.java | 2 +- .../OzoneDelegationTokenSecretManager.java | 2 +- .../main/resources/webapps/ozoneManager/index.html | 4 +- .../main/resources/webapps/ozoneManager/main.html | 2 + .../webapps/ozoneManager/om-overview.html | 26 + .../resources/webapps/ozoneManager/ozoneManager.js | 7 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 43 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 79 +- .../hadoop/ozone/om/failover/TestOMFailovers.java | 151 ++ ...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +- .../ozone/om/request/TestNormalizePaths.java | 109 ++ .../ozone/om/request/TestOMRequestUtils.java | 130 +- .../bucket/TestOMBucketSetPropertyRequest.java | 41 +- .../request/file/TestOMDirectoryCreateRequest.java | 6 + .../om/request/key/TestOMAllocateBlockRequest.java | 22 +- .../om/request/key/TestOMKeyCreateRequest.java | 182 +- .../key/TestOMKeyPurgeRequestAndResponse.java | 15 +- .../ozone/om/request/key/TestOMKeyRequest.java | 17 + .../om/request/key/TestOMKeysRenameRequest.java | 160 ++ .../request/key/TestOMOpenKeysDeleteRequest.java | 419 +++++ .../TestS3InitiateMultipartUploadRequest.java | 59 +- .../s3/multipart/TestS3MultipartRequest.java | 17 +- .../TestS3MultipartUploadCommitPartRequest.java | 2 +- .../volume/TestOMVolumeSetOwnerRequest.java | 2 +- .../volume/TestOMVolumeSetQuotaRequest.java | 69 +- .../bucket/TestOMBucketCreateResponse.java | 8 + .../bucket/TestOMBucketDeleteResponse.java | 8 + .../bucket/TestOMBucketSetPropertyResponse.java | 9 + .../file/TestOMDirectoryCreateResponse.java | 8 + .../response/key/TestOMAllocateBlockResponse.java | 27 +- .../om/response/key/TestOMKeyCommitResponse.java | 19 +- .../om/response/key/TestOMKeyCreateResponse.java | 23 +- .../om/response/key/TestOMKeyDeleteResponse.java | 27 +- .../ozone/om/response/key/TestOMKeyResponse.java | 3 + .../om/response/key/TestOMKeysDeleteResponse.java | 38 +- .../om/response/key/TestOMKeysRenameResponse.java | 131 ++ .../response/key/TestOMOpenKeysDeleteResponse.java | 185 ++ .../s3/multipart/TestS3MultipartResponse.java | 16 +- .../TestS3MultipartUploadAbortResponse.java | 21 +- .../security/TestOMDelegationTokenResponse.java | 8 + .../volume/TestOMVolumeCreateResponse.java | 8 + .../volume/TestOMVolumeDeleteResponse.java | 8 + .../volume/TestOMVolumeSetOwnerResponse.java | 9 + .../volume/TestOMVolumeSetQuotaResponse.java | 9 + .../ozone/security/TestOzoneTokenIdentifier.java | 19 + hadoop-ozone/ozonefs-common/pom.xml | 4 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 23 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 143 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 138 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 180 +- .../hadoop/fs/ozone/OzoneClientAdapterImpl.java | 4 +- .../fs/ozone/RootedOzoneClientAdapterImpl.java | 4 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/ozonefs-hadoop2/pom.xml | 6 +- .../org/apache/hadoop/fs/ozone/RootedOzFs.java | 49 + .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/ozonefs-hadoop3/pom.xml | 6 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +- .../org/apache/hadoop/fs/ozone/RootedOzFs.java | 44 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 4 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/ozonefs-shaded/pom.xml | 9 +- hadoop-ozone/ozonefs/pom.xml | 4 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 4 +- .../org/apache/hadoop/fs/ozone/RootedOzFs.java | 44 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 4 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/pom.xml | 33 +- hadoop-ozone/recon-codegen/pom.xml | 2 +- .../recon/schema/ContainerSchemaDefinition.java | 4 +- .../recon/schema/ReconTaskSchemaDefinition.java | 2 +- .../ozone/recon/schema/StatsSchemaDefinition.java | 14 +- .../recon/schema/UtilizationSchemaDefinition.java | 23 +- hadoop-ozone/recon/pom.xml | 2 +- .../ozone/recon/MetricsServiceProviderFactory.java | 86 + .../apache/hadoop/ozone/recon/ReconConstants.java | 8 +- .../hadoop/ozone/recon/ReconControllerModule.java | 13 +- .../hadoop/ozone/recon/ReconSchemaManager.java | 2 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 50 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 47 +- .../ozone/recon/api/ClusterStateEndpoint.java | 47 +- .../hadoop/ozone/recon/api/ContainerEndpoint.java | 2 +- .../ozone/recon/api/MetricsProxyEndpoint.java | 118 ++ .../hadoop/ozone/recon/api/NodeEndpoint.java | 2 + .../hadoop/ozone/recon/api/PipelineEndpoint.java | 59 +- .../ozone/recon/api/types/DatanodeMetadata.java | 28 + .../ozone/recon/codec/DatanodeDetailsCodec.java | 50 + .../ozone/recon/codec/ReconNodeDBKeyCodec.java | 46 + .../hadoop/ozone/recon/codec/package-info.java | 22 + .../apache/hadoop/ozone/recon/metrics/Metric.java | 51 + .../ozone/recon/scm/ReconContainerManager.java | 61 +- .../recon/scm/ReconContainerReportHandler.java | 2 +- .../hadoop/ozone/recon/scm/ReconDBDefinition.java | 38 - .../ReconIncrementalContainerReportHandler.java | 6 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 55 +- .../ozone/recon/scm/ReconSCMDBDefinition.java | 61 + .../scm/ReconStorageContainerManagerFacade.java | 9 +- .../ozone/recon/spi/MetricsServiceProvider.java | 60 + .../spi/impl/ContainerDBServiceProviderImpl.java | 60 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 54 +- .../spi/impl/PrometheusServiceProviderImpl.java | 213 +++ .../recon/spi/impl/ReconContainerDBProvider.java | 28 +- .../ozone/recon/spi/impl/ReconDBDefinition.java | 71 + .../ozone/recon/tasks/FileSizeCountTask.java | 4 +- .../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 92 +- .../ozone/recon/tasks/ReconTaskControllerImpl.java | 10 +- .../hadoop/ozone/recon/tasks/TableCountTask.java | 185 ++ .../webapps/recon/ozone-recon-web/api/db.json | 48 +- .../components/autoReloadPanel/autoReloadPanel.tsx | 4 +- .../ozone-recon-web/src/types/datanode.types.tsx | 4 +- .../ozone-recon-web/src/utils/columnSearch.less | 36 + .../ozone-recon-web/src/utils/columnSearch.tsx | 94 + .../src/views/datanodes/datanodes.less | 1 + .../src/views/datanodes/datanodes.tsx | 101 +- .../src/views/pipelines/pipelines.tsx | 32 +- .../hadoop/ozone/recon/ReconTestInjector.java | 3 + .../apache/hadoop/ozone/recon/TestReconUtils.java | 6 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 151 +- .../scm/AbstractReconContainerManagerTest.java | 51 +- .../ozone/recon/scm/TestReconContainerManager.java | 95 +- ...TestReconIncrementalContainerReportHandler.java | 63 + .../ozone/recon/scm/TestReconNodeManager.java | 20 +- .../ozone/recon/scm/TestReconPipelineManager.java | 8 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 9 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 54 +- .../recon/tasks/TestReconTaskControllerImpl.java | 6 +- .../ozone/recon/tasks/TestTableCountTask.java | 178 ++ .../test/resources/prometheus-test-response.txt | 21 + hadoop-ozone/s3gateway/pom.xml | 9 +- .../hadoop/ozone/s3/AWSSignatureProcessor.java | 435 +++++ .../hadoop/ozone/s3/AWSV4SignatureProcessor.java | 426 ----- .../hadoop/ozone/s3/OzoneClientProducer.java | 8 +- .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 6 + .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 8 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 69 +- .../ozone/s3/header/AuthorizationHeaderV2.java | 2 +- .../ozone/s3/header/AuthorizationHeaderV4.java | 4 +- .../hadoop/ozone/s3/io/S3WrapperInputStream.java | 36 +- .../apache/hadoop/ozone/s3/util/RangeHeader.java | 6 +- .../hadoop/ozone/client/ObjectStoreStub.java | 5 +- .../hadoop/ozone/client/OzoneVolumeStub.java | 6 +- .../hadoop/ozone/s3/TestAWSSignatureProcessor.java | 141 ++ .../ozone/s3/TestAWSV4SignatureProcessor.java | 103 - .../hadoop/ozone/s3/endpoint/TestBucketHead.java | 11 +- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 2 +- .../s3/endpoint/TestMultipartUploadWithCopy.java | 15 +- .../ozone/s3/header/TestAuthorizationHeaderV4.java | 2 +- hadoop-ozone/tools/pom.xml | 18 +- .../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 120 -- .../ozone/admin/om/GetServiceRolesSubcommand.java | 4 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 12 +- .../org/apache/hadoop/ozone/conf/OzoneGetConf.java | 86 + .../ozone/conf/OzoneManagersCommandHandler.java | 53 + .../ozone/conf/PrintConfKeyCommandHandler.java | 52 + .../StorageContainerManagersCommandHandler.java | 52 + .../org/apache/hadoop/ozone/conf/package-info.java | 21 + .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 165 +- .../hadoop/ozone/debug/ContainerChunkInfo.java | 21 +- .../hadoop/ozone/debug/DBDefinitionFactory.java | 22 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 131 +- .../org/apache/hadoop/ozone/debug/DropTable.java | 81 + .../org/apache/hadoop/ozone/debug/ListTables.java | 19 +- .../org/apache/hadoop/ozone/debug/OzoneDebug.java | 11 +- .../org/apache/hadoop/ozone/debug/RDBParser.java | 38 +- .../apache/hadoop/ozone/debug/RocksDBUtils.java | 49 + .../hadoop/ozone/freon/BaseFreonGenerator.java | 22 + .../hadoop/ozone/freon/ContentGenerator.java | 31 +- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 244 +++ .../java/org/apache/hadoop/ozone/freon/Freon.java | 5 + .../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 55 +- .../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +- .../ozone/freon/HadoopNestedDirGenerator.java | 27 +- .../hadoop/ozone/freon/RandomKeyGenerator.java | 53 +- .../apache/hadoop/ozone/fsck/ContainerMapper.java | 2 +- .../hadoop/ozone/segmentparser/RatisLogParser.java | 12 +- .../hadoop/ozone/shell/ClearSpaceQuotaOptions.java | 43 + .../apache/hadoop/ozone/shell/OzoneAddress.java | 103 +- .../org/apache/hadoop/ozone/shell/OzoneShell.java | 14 +- .../hadoop/ozone/shell/SetSpaceQuotaOptions.java | 44 + .../java/org/apache/hadoop/ozone/shell/Shell.java | 8 +- .../hadoop/ozone/shell/bucket/BucketCommands.java | 17 +- .../ozone/shell/bucket/ClearQuotaHandler.java | 55 + .../ozone/shell/bucket/CreateBucketHandler.java | 13 + .../ozone/shell/bucket/LinkBucketHandler.java | 79 + .../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 62 + .../hadoop/ozone/shell/keys/KeyCommands.java | 12 +- .../hadoop/ozone/shell/token/GetTokenHandler.java | 8 +- .../ozone/shell/token/PrintTokenHandler.java | 3 +- .../ozone/shell/token/RenewTokenHandler.java | 4 +- .../hadoop/ozone/shell/token/TokenCommands.java | 16 +- .../hadoop/ozone/shell/token/TokenOption.java | 38 +- .../ozone/shell/volume/ClearQuotaHandler.java | 53 + .../ozone/shell/volume/CreateVolumeHandler.java | 20 +- .../hadoop/ozone/shell/volume/SetQuotaHandler.java | 61 + .../ozone/shell/volume/UpdateVolumeHandler.java | 11 - .../hadoop/ozone/shell/volume/VolumeCommands.java | 16 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + .../hadoop/ozone/conf/TestGetConfOptions.java | 90 + .../ozone/debug/TestDBDefinitionFactory.java | 59 + .../hadoop/ozone/freon/TestContentGenerator.java | 82 + .../TestGenerateOzoneRequiredConfigurations.java | 5 +- .../hadoop/ozone/shell/TestOzoneAddress.java | 6 +- .../shell/TestOzoneAddressClientCreation.java | 172 ++ hadoop-ozone/upgrade/pom.xml | 57 - .../org/apache/hadoop/ozone/upgrade/Balance.java | 38 - .../org/apache/hadoop/ozone/upgrade/Execute.java | 37 - .../hadoop/ozone/upgrade/InPlaceUpgrade.java | 45 - .../java/org/apache/hadoop/ozone/upgrade/Plan.java | 38 - .../apache/hadoop/ozone/upgrade/package-info.java | 23 - pom.xml | 93 +- 999 files changed, 37896 insertions(+), 15714 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 8ed5ab6,a7aca16..03da6dd --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@@ -365,11 -379,6 +379,13 @@@ public final class OzoneConsts public static final String CONTAINER_DB_TYPE_ROCKSDB = "RocksDB"; public static final String CONTAINER_DB_TYPE_LEVELDB = "LevelDB"; + // SCM HA + public static final String SCM_SERVICE_ID_DEFAULT = "scmServiceIdDefault"; + + // SCM Ratis snapshot file to store the last applied index + public static final String SCM_RATIS_SNAPSHOT_INDEX = "scmRatisSnapshotIndex"; + + public static final String SCM_RATIS_SNAPSHOT_TERM = "scmRatisSnapshotTerm"; + // An on-disk transient marker file used when replacing DB with checkpoint + public static final String DB_TRANSIENT_MARKER = "dbInconsistentMarker"; } diff --cc hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java index eecb512,653300e..816e1b9 --- a/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java +++ b/hadoop-hdds/config/src/main/java/org/apache/hadoop/hdds/conf/ConfigurationReflectionUtil.java @@@ -92,9 -94,9 +94,12 @@@ public final class ConfigurationReflect forcedFieldSet(field, configuration, from.getTimeDuration(key, "0s", configAnnotation.timeUnit())); break; + case SIZE: + forcedFieldSet(field, configuration, + from.getStorageSize(key, "0B", configAnnotation.sizeUnit())); + case CLASS: + forcedFieldSet(field, configuration, + from.getClass(key, Object.class)); break; default: throw new ConfigurationException( diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 9bf687d,3cf12e7..cbd713c --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -50,14 -42,8 +50,15 @@@ import org.apache.hadoop.hdds.conf.Conf import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState; + import org.apache.hadoop.hdds.scm.PipelineChoosePolicy; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.ha.SCMHAManager; +import org.apache.hadoop.hdds.scm.ha.SCMHAManagerImpl; +import org.apache.hadoop.hdds.scm.ha.SCMHAUtils; +import org.apache.hadoop.hdds.scm.ha.SCMNodeDetails; +import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisServer; +import org.apache.hadoop.hdds.scm.server.ratis.SCMRatisSnapshotInfo; +import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.block.BlockManager; @@@ -96,7 -82,8 +97,8 @@@ import org.apache.hadoop.hdds.scm.node. import org.apache.hadoop.hdds.scm.pipeline.PipelineActionHandler; import org.apache.hadoop.hdds.scm.pipeline.PipelineManager; import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler; -import org.apache.hadoop.hdds.scm.pipeline.SCMPipelineManager; +import org.apache.hadoop.hdds.scm.pipeline.PipelineManagerV2Impl; + import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; import org.apache.hadoop.hdds.security.exception.SCMSecurityException; import org.apache.hadoop.hdds.security.x509.SecurityConfig; @@@ -1175,56 -1119,6 +1180,56 @@@ public final class StorageContainerMana return this.clusterMap; } + private static SCMRatisServer initializeRatisServer( + OzoneConfiguration conf, StorageContainerManager scm) throws IOException { + SCMNodeDetails scmNodeDetails = SCMNodeDetails + .initStandAlone(conf); + //TODO enable Ratis group + SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer( + conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class), + scm, scmNodeDetails, Collections.EMPTY_LIST, + SCMRatisServer.getSCMRatisDirectory(conf)); + if (scmRatisServer != null) { + LOG.info("SCM Ratis server initialized at port {}", + scmRatisServer.getServerPort()); + } // TODO error handling for scmRatisServer creation failure + return scmRatisServer; + } + + @VisibleForTesting + public SCMRatisServer getScmRatisServer() { + return scmRatisServer; + } + + public void setScmRatisServer(SCMRatisServer scmRatisServer) { + this.scmRatisServer = scmRatisServer; + } + + @VisibleForTesting + public SCMRatisSnapshotInfo getSnapshotInfo() { + return scmRatisSnapshotInfo; + } + + @VisibleForTesting + public long getRatisSnapshotIndex() { + return scmRatisSnapshotInfo.getIndex(); + } + + /** + * Save ratis snapshot to SCM meta store and local disk. + */ + public TermIndex saveRatisSnapshot() throws IOException { + TermIndex snapshotIndex = scmRatisServer.getLastAppliedTermIndex(); + if (scmMetadataStore != null) { + // Flush the SCM state to disk - scmMetadataStore.getStore().flush(); ++ scmMetadataStore.getStore().flushDB(); + } + + scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex); + + return snapshotIndex; + } + /** * Get the safe mode status of all rules. * diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java index 597a317,2b492a2..90f4996 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManager.java @@@ -26,9 -26,6 +26,12 @@@ import static org.apache.hadoop.hdds.Hd import static org.apache.hadoop.hdds.HddsConfigKeys .HDDS_SCM_SAFEMODE_PIPELINE_CREATION; import static org.junit.Assert.fail; ++<<<<<<< HEAD + +import org.apache.hadoop.hdds.scm.TestUtils; +import org.junit.Ignore; ++======= ++>>>>>>> master import static org.mockito.Matchers.argThat; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java index 18b9c4c,7a28ba5..3ac5ad8 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestFreonWithDatanodeRestart.java @@@ -74,15 -74,13 +74,12 @@@ public class TestFreonWithDatanodeResta ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3)); ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3)); conf.setFromObject(ratisServerConfig); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "rpc.request.timeout", - 3, TimeUnit.SECONDS); - conf.setTimeDuration( - RatisHelper.HDDS_DATANODE_RATIS_CLIENT_PREFIX_KEY+ "." + - "watch.request.timeout", - 3, TimeUnit.SECONDS); - conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 5); - + RatisClientConfig.RaftConfig raftClientConfig = + conf.getObject(RatisClientConfig.RaftConfig.class); + raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3)); + raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3)); + conf.setFromObject(raftClientConfig); + cluster = MiniOzoneCluster.newBuilder(conf) .setHbProcessorInterval(1000) .setHbInterval(1000) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
