This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-2665-ofs in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit dd992234b9d3a585dd6bc6119436075a80b2ca8f Merge: 8356a4a 266a9e8 Author: Siyao Meng <[email protected]> AuthorDate: Thu Apr 23 10:57:47 2020 -0700 Merge remote-tracking branch 'asf/master' into HDDS-2665-ofs Conflicts: hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot .github/comment-commands/close.sh | 41 ++ .github/comment-commands/pending.sh | 7 + .github/comment-commands/ready.sh | 6 + .github/workflows/post-commit.yml | 68 +- .github/workflows/pr.yml | 68 +- CONTRIBUTION.md => CONTRIBUTING.md | 0 HISTORY.md | 61 ++ README.md | 6 +- dev-support/bin/qbt | 18 - dev-support/bin/smart-apply-patch | 18 - dev-support/bin/test-patch | 18 - dev-support/bin/yetus-wrapper | 188 ----- dev-support/byteman/mock-scm.btm | 34 + dev-support/byteman/ratis-mock-followers.btm | 42 ++ hadoop-hdds/client/pom.xml | 13 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 84 ++- .../hadoop/hdds/scm/XceiverClientManager.java | 10 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 57 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 49 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 15 +- hadoop-hdds/common/pom.xml | 61 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 6 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 25 +- .../hadoop/hdds/annotation/InterfaceStability.java | 3 - .../org/apache/hadoop/hdds/cli/GenericCli.java | 12 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 193 +---- .../hadoop/hdds/freon/FakeClusterTopology.java | 92 +++ .../freon/FakeScmBlockLocationProtocolClient.java | 100 +++ .../FakeScmContainerLocationProtocolClient.java | 76 ++ .../apache/hadoop/hdds/freon}/package-info.java | 14 +- .../java/org/apache/hadoop/hdds/fs/DUFactory.java | 11 +- .../hdds/fs/DedicatedDiskSpaceUsageFactory.java | 11 +- .../hadoop/hdds/fs/SpaceUsageCheckFactory.java | 26 +- .../hdds/function/SupplierWithIOException.java | 19 +- .../hdds/ratis/ContainerCommandRequestMessage.java | 7 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 32 +- .../hadoop/hdds/scm/ByteStringConversion.java | 4 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 24 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +- .../hadoop/hdds/scm/container/ContainerInfo.java | 56 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 4 +- .../hadoop/hdds/scm/net/NodeSchemaManager.java | 4 +- .../protocol/StorageContainerLocationProtocol.java | 12 + ...inerLocationProtocolClientSideTranslatorPB.java | 42 +- .../hadoop/hdds/security/x509/SecurityConfig.java | 22 +- .../hadoop/hdds/tracing/GrpcServerInterceptor.java | 11 +- .../apache/hadoop/hdds/tracing/TraceAllMethod.java | 8 +- .../apache/hadoop/hdds/tracing/TracingUtil.java | 71 +- .../utils/LegacyHadoopConfigurationSource.java | 74 ++ .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 30 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 7 +- .../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 30 +- .../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +- .../org/apache/hadoop/ozone/lock/LockManager.java | 6 +- .../java/org/apache/hadoop/ozone/package-info.java | 21 +- .../proto/StorageContainerLocationProtocol.proto | 12 + .../common/src/main/resources/ozone-default.xml | 82 ++- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 7 +- .../hadoop/hdds/conf/TestOzoneConfiguration.java | 18 +- .../org/apache/hadoop/hdds/fs/TestDUFactory.java | 8 +- .../fs/TestDedicatedDiskSpaceUsageFactory.java | 8 +- .../hadoop/hdds/fs/TestSpaceUsageFactory.java | 40 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 38 +- .../hadoop/hdds/scm/net/TestNodeSchemaManager.java | 16 +- hadoop-hdds/config/pom.xml | 13 +- .../hdds/conf/ConfigurationReflectionUtil.java | 159 ++++ .../hadoop/hdds/conf/ConfigurationSource.java | 291 ++++++++ .../org/apache/hadoop/hdds/conf/StorageSize.java | 102 +++ .../org/apache/hadoop/hdds/conf/StorageUnit.java | 529 +++++++++++++ .../apache/hadoop/hdds/conf/TimeDurationUtil.java | 154 ++++ hadoop-hdds/container-service/pom.xml | 17 +- .../hadoop/hdds/freon/FakeRatisFollower.java | 125 ++++ .../apache/hadoop/hdds/freon}/package-info.java | 8 +- .../hadoop/ozone/HddsDatanodeHttpServer.java | 4 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 4 +- .../container/common/helpers/ContainerMetrics.java | 4 +- .../container/common/impl/ChunkLayOutVersion.java | 62 +- .../ozone/container/common/impl/ContainerData.java | 19 + .../container/common/impl/HddsDispatcher.java | 87 ++- .../ozone/container/common/interfaces/Handler.java | 8 +- .../container/common/report/ReportManager.java | 21 +- .../container/common/report/ReportPublisher.java | 27 +- .../common/report/ReportPublisherFactory.java | 32 +- .../common/statemachine/DatanodeStateMachine.java | 51 +- .../common/statemachine/EndpointStateMachine.java | 6 +- .../common/statemachine/SCMConnectionManager.java | 63 +- .../common/statemachine/StateContext.java | 62 +- .../CreatePipelineCommandHandler.java | 29 +- .../commandhandler/DeleteBlocksCommandHandler.java | 6 +- .../ReplicateContainerCommandHandler.java | 6 +- .../common/states/datanode/InitDatanodeState.java | 32 +- .../states/datanode/RunningDatanodeState.java | 6 +- .../states/endpoint/HeartbeatEndpointTask.java | 10 +- .../states/endpoint/RegisterEndpointTask.java | 10 +- .../states/endpoint/VersionEndpointTask.java | 23 +- .../common/transport/server/XceiverServerGrpc.java | 39 +- .../server/ratis/ContainerStateMachine.java | 110 ++- .../transport/server/ratis/XceiverServerRatis.java | 118 +-- .../container/common/utils/ContainerCache.java | 39 +- .../ozone/container/common/volume/HddsVolume.java | 25 +- .../container/common/volume/HddsVolumeChecker.java | 4 +- .../container/common/volume/MutableVolumeSet.java | 13 +- .../ozone/container/common/volume/VolumeInfo.java | 17 +- .../container/keyvalue/KeyValueContainer.java | 49 +- .../container/keyvalue/KeyValueContainerCheck.java | 133 ++-- .../container/keyvalue/KeyValueContainerData.java | 24 +- .../ozone/container/keyvalue/KeyValueHandler.java | 115 +-- .../container/keyvalue/helpers/BlockUtils.java | 36 +- .../container/keyvalue/helpers/ChunkUtils.java | 39 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 23 +- .../container/keyvalue/impl/BlockManagerImpl.java | 6 +- .../keyvalue/impl/ChunkManagerFactory.java | 4 +- .../keyvalue/impl/FilePerBlockStrategy.java | 23 +- .../keyvalue/impl/FilePerChunkStrategy.java | 25 +- .../keyvalue/interfaces/ChunkManager.java | 1 - .../background/BlockDeletingService.java | 81 +- .../ozone/container/ozoneimpl/ContainerReader.java | 30 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 9 +- .../replication/ContainerStreamingOutput.java | 44 -- .../replication/SimpleContainerDownloader.java | 4 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 3 +- .../ozone/container/common/ContainerTestUtils.java | 24 +- .../ozone/container/common/SCMTestUtils.java | 23 +- .../container/common/TestBlockDeletingService.java | 80 +- .../ozone/container/common/TestContainerCache.java | 2 - .../container/common/TestDatanodeStateMachine.java | 60 +- .../common/TestKeyValueContainerData.java | 18 +- .../common/impl/TestContainerDataYaml.java | 21 +- .../impl/TestContainerDeletionChoosingPolicy.java | 21 +- .../common/impl/TestContainerPersistence.java | 98 +-- .../container/common/impl/TestContainerSet.java | 23 +- .../container/common/impl/TestHddsDispatcher.java | 23 +- .../container/common/interfaces/TestHandler.java | 19 +- .../container/common/report/TestReportManager.java | 11 +- .../common/report/TestReportPublisher.java | 4 +- .../common/report/TestReportPublisherFactory.java | 14 +- .../TestCloseContainerCommandHandler.java | 21 +- .../states/endpoint/TestHeartbeatEndpointTask.java | 52 +- .../container/common/volume/TestHddsVolume.java | 28 +- .../common/volume/TestHddsVolumeChecker.java | 40 +- .../volume/TestRoundRobinVolumeChoosingPolicy.java | 22 +- .../common/volume/TestVolumeSetDiskChecks.java | 16 +- .../container/keyvalue/ChunkLayoutTestInfo.java | 120 +++ .../keyvalue/TestKeyValueBlockIterator.java | 58 +- .../container/keyvalue/TestKeyValueContainer.java | 23 +- .../keyvalue/TestKeyValueContainerCheck.java | 76 +- .../TestKeyValueContainerMarkUnhealthy.java | 16 +- .../container/keyvalue/TestKeyValueHandler.java | 164 +++-- .../container/keyvalue/TestTarContainerPacker.java | 18 +- .../keyvalue/impl/AbstractTestChunkManager.java | 170 +++++ .../keyvalue/impl/CommonChunkManagerTestCases.java | 201 +++++ .../keyvalue/impl/TestBlockManagerImpl.java | 17 +- .../container/keyvalue/impl/TestChunkManager.java | 325 -------- .../keyvalue/impl/TestChunkManagerDummyImpl.java | 60 ++ .../keyvalue/impl/TestFilePerBlockStrategy.java | 140 ++++ .../keyvalue/impl/TestFilePerChunkStrategy.java | 92 +++ .../container/ozoneimpl/TestOzoneContainer.java | 19 +- .../replication/TestReplicationSupervisor.java | 32 +- .../testutils/BlockDeletingServiceTestImpl.java | 16 +- hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 10 +- hadoop-hdds/docs/archetypes/design.md | 60 ++ hadoop-hdds/docs/content/beyond/Containers.zh.md | 203 +++++ hadoop-hdds/docs/content/concept/Datanodes.md | 4 +- hadoop-hdds/docs/content/concept/Datanodes.zh.md | 30 +- hadoop-hdds/docs/content/design/configless.md | 31 + hadoop-hdds/docs/content/design/decommissioning.md | 17 +- hadoop-hdds/docs/content/design/gdpr.md | 31 + hadoop-hdds/docs/content/design/locks.md | 29 + hadoop-hdds/docs/content/design/multiraft.md | 31 + .../docs/content/design/namespace-support.md | 29 + hadoop-hdds/docs/content/design/nfs.md | 31 + hadoop-hdds/docs/content/design/ofs.md | 33 + hadoop-hdds/docs/content/design/omha.md | 33 + .../content/design/ozone-enhancement-proposals.md | 2 +- .../docs/content/design/ozone-volume-management.md | 181 +++++ hadoop-hdds/docs/content/design/recon1.md | 32 + hadoop-hdds/docs/content/design/recon2.md | 29 + hadoop-hdds/docs/content/design/s3gateway.md | 31 + hadoop-hdds/docs/content/design/scmha.md | 29 + hadoop-hdds/docs/content/design/tde.md | 29 + hadoop-hdds/docs/content/design/token.md | 25 + hadoop-hdds/docs/content/design/trash.md | 25 + hadoop-hdds/docs/content/design/typesafeconfig.md | 33 + hadoop-hdds/docs/content/recipe/Prometheus.zh.md | 93 +++ .../{tools/SCMCLI.md => recipe/_index.zh.md} | 15 +- hadoop-hdds/docs/content/shell/VolumeCommands.md | 6 +- .../docs/content/shell/VolumeCommands.zh.md | 5 +- hadoop-hdds/docs/content/tools/Admin.md | 35 + hadoop-hdds/docs/content/tools/AuditParser.zh.md | 68 ++ .../Datanodes.zh.md => tools/Genconf.zh.md} | 11 +- hadoop-hdds/docs/content/tools/TestTools.md | 8 +- .../tools/{TestTools.md => TestTools.zh.md} | 53 +- hadoop-hdds/docs/content/tools/_index.md | 4 +- hadoop-hdds/docs/pom.xml | 4 +- .../ozonedoc/layouts/_default/baseof.html} | 31 +- .../themes/ozonedoc/layouts/design/section.html | 68 ++ .../themes/ozonedoc/layouts/design/single.html | 47 ++ .../themes/ozonedoc/layouts/partials/sidebar.html | 1 + hadoop-hdds/framework/pom.xml | 49 +- .../hadoop/hdds/conf/DatanodeRatisGrpcConfig.java | 0 .../hdds/conf/DatanodeRatisServerConfig.java | 0 .../apache/hadoop/hdds/conf/HddsConfServlet.java | 30 +- .../org/apache/hadoop/hdds/conf}/package-info.java | 5 +- .../certificates/utils/CertificateSignRequest.java | 23 +- .../certificates/utils/SelfSignedCertificate.java | 27 +- .../hdds/security/x509/keys/HDDSKeyGenerator.java | 13 +- .../org/apache/hadoop/hdds/server}/JsonUtils.java | 8 +- .../server/OzoneProtocolMessageDispatcher.java | 18 +- .../org/apache/hadoop/hdds/server/ServerUtils.java | 12 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 36 +- .../hadoop/hdds/server/http/FilterInitializer.java | 4 +- .../apache/hadoop/hdds/server/http/HttpConfig.java | 4 +- .../hadoop/hdds/server/http/HttpServer2.java | 98 +-- .../server/http/RatisNameRewriteSampleBuilder.java | 17 +- .../hdds/server/http/StaticUserWebFilter.java | 6 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 39 +- .../hadoop/hdds/utils/MetadataStoreBuilder.java | 13 +- .../hadoop/hdds/utils/ProtocolMessageMetrics.java | 42 +- .../hdds/utils/db/BatchOperationHandler.java | 38 +- .../hdds/utils/db/DBColumnFamilyDefinition.java | 81 ++ .../apache/hadoop/hdds/utils/db/DBDefinition.java} | 39 +- .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 1 - .../org/apache/hadoop/hdds/utils/db/DBStore.java | 18 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 56 +- .../apache/hadoop/hdds/utils/db/RDBMetrics.java | 13 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 2 +- .../apache/hadoop/hdds/server/TestJsonUtils.java | 37 +- .../apache/hadoop/hdds/server/TestServerUtils.java | 21 +- .../hdds/server/http/TestBaseHttpServer.java | 4 +- .../hdds/server/http/TestRatisNameRewrite.java | 10 + .../hadoop/hdds/utils/TestMetadataStore.java | 13 +- .../hadoop/hdds/utils/TestRocksDBStoreMBean.java | 23 +- hadoop-hdds/hadoop-dependency-client/README.md | 60 ++ hadoop-hdds/hadoop-dependency-client/pom.xml | 282 +++++++ .../{tools => hadoop-dependency-server}/pom.xml | 49 +- .../{client => hadoop-dependency-test}/pom.xml | 26 +- hadoop-hdds/pom.xml | 34 +- hadoop-hdds/server-scm/pom.xml | 26 +- .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 35 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 33 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 34 +- .../hdds/scm/block/SCMBlockDeletingService.java | 41 +- .../hdds/scm/container/ContainerManager.java | 6 +- .../hdds/scm/container/ContainerReportHandler.java | 57 +- .../hdds/scm/container/ContainerStateManager.java | 61 +- .../hdds/scm/container/ReplicationManager.java | 9 +- .../hdds/scm/container/SCMContainerManager.java | 145 ++-- .../ContainerPlacementPolicyFactory.java | 11 +- .../algorithms/SCMContainerPlacementCapacity.java | 4 +- .../algorithms/SCMContainerPlacementRackAware.java | 4 +- .../algorithms/SCMContainerPlacementRandom.java | 4 +- .../container/placement/metrics/ContainerStat.java | 2 +- .../hadoop/hdds/scm/metadata/ContainerIDCodec.java | 48 ++ .../hdds/scm/metadata/ContainerInfoCodec.java | 47 ++ .../hadoop/hdds/scm/metadata/PipelineCodec.java | 56 ++ .../hadoop/hdds/scm/metadata/PipelineIDCodec.java | 45 ++ .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 98 +++ .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 27 +- .../hdds/scm/metadata/SCMMetadataStoreRDBImpl.java | 113 +-- .../hadoop/hdds/scm/node/NewNodeHandler.java | 15 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 55 +- .../scm/node/NonHealthyToHealthyNodeHandler.java | 4 +- .../hadoop/hdds/scm/node/StaleNodeHandler.java | 4 +- .../scm/pipeline/BackgroundPipelineCreator.java | 15 +- .../hdds/scm/pipeline/PipelineActionHandler.java | 63 +- .../hadoop/hdds/scm/pipeline/PipelineFactory.java | 6 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 140 ++-- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 75 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 6 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 76 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 11 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 178 +++-- .../hdds/scm/pipeline/SCMPipelineMetrics.java | 12 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 15 +- ...lockLocationProtocolServerSideTranslatorPB.java | 4 +- ...inerLocationProtocolServerSideTranslatorPB.java | 27 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +- .../hdds/scm/safemode/DataNodeSafeModeRule.java | 4 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 11 +- .../safemode/OneReplicaPipelineSafeModeRule.java | 4 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 60 +- .../hadoop/hdds/scm/safemode/SafeModeHandler.java | 23 +- .../hadoop/hdds/scm/safemode/SafeModePrecheck.java | 4 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 6 +- .../hdds/scm/server/SCMClientProtocolServer.java | 185 +++-- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 21 +- .../hdds/scm/server/StorageContainerManager.java | 88 ++- .../server/StorageContainerManagerHttpServer.java | 8 +- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 18 +- .../hadoop/hdds/scm/TestHddsServerUtils.java | 42 +- .../scm/TestStorageContainerManagerHttpServer.java | 6 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 37 +- .../container/TestCloseContainerEventHandler.java | 38 +- .../scm/container/TestContainerReportHandler.java | 4 +- .../TestIncrementalContainerReportHandler.java | 4 +- .../hdds/scm/container/TestReplicationManager.java | 34 +- .../scm/container/TestSCMContainerManager.java | 65 +- .../scm/container/TestUnknownContainerReport.java | 145 ++++ .../algorithms/TestContainerPlacementFactory.java | 19 +- .../TestSCMContainerPlacementCapacity.java | 4 +- .../TestSCMContainerPlacementRackAware.java | 4 +- .../TestSCMContainerPlacementRandom.java | 4 +- .../hdds/scm/node/TestContainerPlacement.java | 65 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 14 +- .../scm/pipeline/MockRatisPipelineProvider.java | 20 +- .../scm/pipeline/TestPipelineActionHandler.java | 69 ++ .../scm/pipeline/TestPipelinePlacementPolicy.java | 93 ++- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 178 ++++- .../scm/pipeline/TestSimplePipelineProvider.java | 2 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 38 +- .../TestOneReplicaPipelineSafeModeRule.java | 19 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 179 ++++- .../hdds/scm/safemode/TestSafeModeHandler.java | 71 +- .../ozone/container/common/TestEndPoint.java | 7 +- .../placement/TestContainerPlacement.java | 32 +- .../upgrade => hadoop-hdds/test-utils}/pom.xml | 50 +- .../org/apache/hadoop/test/GenericTestUtils.java | 365 +++++++++ .../org/apache/hadoop/test/LambdaTestUtils.java | 814 +++++++++++++++++++++ .../apache/hadoop/test/TimedOutTestsListener.java | 183 +++++ .../java/org/apache/hadoop/test}/package-info.java | 7 +- hadoop-hdds/tools/pom.xml | 13 +- .../hdds/scm/cli/ContainerOperationClient.java | 24 +- .../hdds/scm/cli/ReplicationManagerCommands.java | 21 +- .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 114 --- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 16 +- .../hadoop/hdds/scm/cli/TopologySubcommand.java | 26 +- .../hdds/scm/cli/container/CloseSubcommand.java | 4 +- .../hdds/scm/cli/container/ContainerCommands.java | 32 +- .../hdds/scm/cli/container/DeleteSubcommand.java | 3 +- .../hdds/scm/cli/container/InfoSubcommand.java | 20 +- .../hdds/scm/cli/container/ListSubcommand.java | 21 +- .../hdds/scm/cli/container/WithScmClient.java | 13 +- .../hdds/scm/cli/datanode/DatanodeCommands.java | 22 +- .../hdds/scm/cli/pipeline/PipelineCommands.java | 22 +- hadoop-ozone/Jenkinsfile | 116 --- hadoop-ozone/client/pom.xml | 9 +- .../apache/hadoop/ozone/client/ObjectStore.java | 6 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 8 +- .../apache/hadoop/ozone/client/OzoneClient.java | 4 +- .../hadoop/ozone/client/OzoneClientFactory.java | 69 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 18 +- .../ozone/client/io/BlockOutputStreamEntry.java | 36 +- .../client/io/BlockOutputStreamEntryPool.java | 8 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 16 +- .../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 32 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 91 ++- .../hadoop/ozone/client/TestHddsClientUtils.java | 41 +- hadoop-ozone/common/pom.xml | 29 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 166 +++-- .../apache/hadoop/ozone/conf/OMClientConfig.java | 67 ++ .../apache/hadoop/ozone/conf}/package-info.java | 11 +- .../apache/hadoop/ozone/freon/OzoneGetConf.java | 27 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 5 +- .../hadoop/ozone/om/exceptions/OMException.java | 2 + .../ozone/om/exceptions/OMReplayException.java | 16 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 105 ++- .../apache/hadoop/ozone/om/helpers/DBUpdates.java | 58 ++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 + .../hadoop/ozone/om/helpers/OzoneFileStatus.java | 116 ++- .../hadoop/ozone/om/helpers/ServiceInfo.java | 24 +- .../hadoop/ozone/om/lock/OzoneManagerLock.java | 4 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 20 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 127 ++-- .../ozone/security/OzoneTokenIdentifier.java | 22 +- .../apache/hadoop/ozone/web/utils/OzoneUtils.java | 7 +- .../common/src/main/proto/FSProtos.proto | 2 +- .../src/main/proto/OzoneManagerProtocol.proto | 6 + .../java/org/apache/hadoop/ozone/TestOmUtils.java | 125 +--- hadoop-ozone/csi/pom.xml | 4 +- hadoop-ozone/datanode/pom.xml | 6 +- hadoop-ozone/dev-support/checks/author.sh | 6 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 14 +- .../intellij/runConfigurations/OzoneShell.xml | 6 +- hadoop-ozone/dist/pom.xml | 16 +- .../main/compose/ozone-hdfs/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-mr/common-config | 2 +- .../compose/ozone-mr/hadoop27/docker-compose.yaml | 1 + .../ozone-mr/hadoop27/{test.sh => run-test.sh} | 0 .../compose/ozone-mr/hadoop31/docker-compose.yaml | 1 + .../src/main/compose/ozone-mr/hadoop31/test.sh | 6 +- .../compose/ozone-mr/hadoop32/docker-compose.yaml | 1 + .../dist/src/main/compose/ozone-om-ha/.ssh/id_rsa | 15 - .../src/main/compose/ozone-om-ha/.ssh/id_rsa.pub | 15 - .../src/main/compose/ozone-om-ha/docker-config | 3 +- .../ozone-om-ha/{run.sh => test_disabled.sh} | 0 .../dist/src/main/compose/ozone-topology/.env | 2 +- .../compose/ozone-topology/docker-compose.yaml | 3 +- .../src/main/compose/ozone-topology/docker-config | 2 +- .../src/main/compose/ozone-topology/hdds-3084.sh | 56 -- .../dist/src/main/compose/ozone-topology/test.sh | 20 +- .../src/main/compose/ozone/docker-compose.yaml | 1 + hadoop-ozone/dist/src/main/compose/ozone/test.sh | 2 + .../main/compose/ozoneblockade/docker-compose.yaml | 1 + .../compose/ozones3-haproxy/docker-compose.yaml | 1 + .../main/compose/ozonescripts/docker-compose.yaml | 1 + .../dist/src/main/compose/ozonesecure-mr/README.md | 12 +- .../compose/ozonesecure-mr/docker-compose.yaml | 1 + .../src/main/compose/ozonesecure-mr/docker-config | 2 +- .../main/compose/ozonesecure/docker-compose.yaml | 1 + .../src/main/compose/ozonesecure/docker-config | 1 + .../dist/src/main/compose/ozonesecure/test.sh | 2 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 24 +- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 8 - .../licenses/LICENSE-org.openjdk.jmh-jmh-core.txt | 347 --------- ...SE-org.openjdk.jmh-jmh-generator-annprocess.txt | 347 --------- .../smoketest/{scmcli => admincli}/datanode.robot | 3 +- .../smoketest/{scmcli => admincli}/pipeline.robot | 5 +- .../main/smoketest/auditparser/auditparser.robot | 1 + .../dist/src/main/smoketest/basic/basic.robot | 1 + .../src/main/smoketest/basic/ozone-shell.robot | 10 +- .../dist/src/main/smoketest/createbucketenv.robot | 2 +- .../dist/src/main/smoketest/createmrenv.robot | 4 +- .../dist/src/main/smoketest/env-compose.robot | 2 +- .../dist/src/main/smoketest/freon/freon.robot | 1 + .../dist/src/main/smoketest/gdpr/gdpr.robot | 1 + .../dist/src/main/smoketest/mapreduce.robot | 4 +- .../main/smoketest/om-ratis/testOMAdminCmd.robot | 2 +- .../dist/src/main/smoketest/omha/testOMHA.robot | 45 +- .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 5 +- .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 5 + .../dist/src/main/smoketest/recon/recon-api.robot | 4 + .../src/main/smoketest/s3/MultipartUpload.robot | 1 + .../dist/src/main/smoketest/s3/awss3.robot | 1 + .../dist/src/main/smoketest/s3/bucketcreate.robot | 1 + .../dist/src/main/smoketest/s3/buckethead.robot | 1 + .../dist/src/main/smoketest/s3/bucketlist.robot | 1 + .../dist/src/main/smoketest/s3/objectcopy.robot | 1 + .../dist/src/main/smoketest/s3/objectdelete.robot | 1 + .../src/main/smoketest/s3/objectmultidelete.robot | 1 + .../dist/src/main/smoketest/s3/objectputget.robot | 1 + .../dist/src/main/smoketest/s3/webui.robot | 1 + .../main/smoketest/security/ozone-secure-fs.robot | 1 + .../main/smoketest/security/ozone-secure-s3.robot | 1 + .../smoketest/security/ozone-secure-token.robot | 76 ++ .../smoketest/topology/{scmcli.robot => cli.robot} | 5 +- .../src/main/smoketest/topology/loaddata.robot | 1 + .../src/main/smoketest/topology/readdata.robot | 1 + .../src/shell/conf/ozone-shell-log4j.properties | 1 + .../dist/src/shell/hdds/hadoop-functions.sh | 63 ++ hadoop-ozone/dist/src/shell/ozone/ozone | 68 +- hadoop-ozone/dist/src/test/shell/gc_opts.bats | 44 ++ .../fault-injection-test/mini-chaos-tests/pom.xml | 14 +- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 252 +++++-- .../ozone/MiniOzoneDatanodeChaosCluster.java | 57 ++ .../hadoop/ozone/MiniOzoneLoadGenerator.java | 14 +- .../hadoop/ozone/MiniOzoneOMChaosCluster.java | 132 ++++ .../hadoop/ozone/TestMiniChaosOzoneCluster.java | 46 +- .../ozone/loadgenerators/AgedLoadGenerator.java | 20 +- .../hadoop/ozone/loadgenerators/DataBuffer.java | 1 + .../loadgenerators/FilesystemLoadGenerator.java | 15 +- .../hadoop/ozone/loadgenerators/LoadExecutors.java | 25 +- .../hadoop/ozone/loadgenerators/LoadGenerator.java | 13 +- .../ozone/loadgenerators/RandomLoadGenerator.java | 10 +- ...adGenerator.java => ReadOnlyLoadGenerator.java} | 49 +- .../org/apache/hadoop/ozone/utils/LoadBucket.java | 17 +- .../fault-injection-test/network-tests/pom.xml | 2 +- hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 30 +- .../apache/hadoop/ozone/insight/LogSubcommand.java | 2 + hadoop-ozone/integration-test/pom.xml | 16 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 112 ++- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 7 - .../ozone/contract/ITestOzoneContractCreate.java | 5 +- .../ozone/contract/ITestOzoneContractDelete.java | 5 +- .../ozone/contract/ITestOzoneContractDistCp.java | 5 +- .../contract/ITestOzoneContractGetFileStatus.java | 5 +- .../fs/ozone/contract/ITestOzoneContractMkdir.java | 5 +- .../fs/ozone/contract/ITestOzoneContractOpen.java | 5 +- .../ozone/contract/ITestOzoneContractRename.java | 5 +- .../ozone/contract/ITestOzoneContractRootDir.java | 6 +- .../fs/ozone/contract/ITestOzoneContractSeek.java | 5 +- .../hadoop/hdds/scm/TestRatisPipelineLeader.java | 6 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 33 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 12 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 157 +++- .../org/apache/hadoop/ozone/RatisTestHelper.java | 3 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 45 +- .../ozone/TestStorageContainerManagerHelper.java | 10 +- .../ozone/client/CertificateClientTestImpl.java | 24 +- .../ozone/client/rpc/Test2BlockOutputStream.java | 173 +++++ .../TestContainerStateMachineFailureOnRead.java | 16 +- .../client/rpc/TestDeleteWithSlowFollower.java | 24 +- .../client/rpc/TestFailureHandlingByClient.java | 23 +- .../client/rpc/TestOzoneAtRestEncryption.java | 23 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 23 +- .../ozone/dn/ratis/TestDnRatisLogParser.java | 21 +- .../ozone/freon/TestOzoneClientKeyGenerator.java | 13 +- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 8 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 37 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 116 ++- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 27 +- .../ozone/om/TestOzoneManagerListVolumes.java | 238 ++++++ .../apache/hadoop/ozone/om/TestScmSafeMode.java | 2 +- .../ozone/om/parser/TestOMRatisLogParser.java | 5 +- .../{ozShell => shell}/TestOzoneDatanodeShell.java | 2 +- .../ozone/{ozShell => shell}/TestOzoneShellHA.java | 6 +- .../ozone/{ozShell => shell}/TestS3Shell.java | 20 +- hadoop-ozone/ozone-manager/pom.xml | 17 +- .../apache/hadoop/ozone/om/KeyDeletingService.java | 4 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 70 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 91 ++- .../apache/hadoop/ozone/om/OMMetadataManager.java | 0 .../java/org/apache/hadoop/ozone/om/OMStorage.java | 4 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 2 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 173 +++-- .../hadoop/ozone/om/OzoneManagerHttpServer.java | 10 +- .../hadoop/ozone/om/S3SecretManagerImpl.java | 0 .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 22 +- .../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 0 .../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 0 .../ozone/om/codec/OmMultipartKeyInfoCodec.java | 0 .../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 0 .../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 0 .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 0 .../hadoop/ozone/om/codec/S3SecretValueCodec.java | 0 .../ozone/om/codec/TokenIdentifierCodec.java | 0 .../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 0 .../apache/hadoop/ozone/om/codec/package-info.java | 0 .../ozone/om/ratis/OzoneManagerRatisServer.java | 37 +- .../om/request/bucket/OMBucketCreateRequest.java | 4 + .../om/request/file/OMDirectoryCreateRequest.java | 42 +- .../ozone/om/request/file/OMFileCreateRequest.java | 13 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 60 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 7 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 80 +- .../request/s3/bucket/S3BucketCreateRequest.java | 13 +- .../S3InitiateMultipartUploadRequest.java | 6 - .../S3MultipartUploadCommitPartRequest.java | 69 +- .../S3MultipartUploadCompleteRequest.java | 78 +- .../om/request/volume/OMVolumeCreateRequest.java | 20 +- .../ozone/om/request/volume/OMVolumeRequest.java | 21 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 26 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 2 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 43 +- .../S3MultipartUploadCommitPartResponse.java | 59 +- .../S3MultipartUploadCompleteResponse.java | 48 +- .../response/volume/OMVolumeSetOwnerResponse.java | 21 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 6 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 13 +- .../protocolPB/OzoneManagerRequestHandler.java | 26 +- .../hadoop/ozone/security/AWSV4AuthValidator.java | 0 .../security/OzoneBlockTokenSecretManager.java | 0 .../OzoneDelegationTokenSecretManager.java | 99 ++- .../hadoop/ozone/security/OzoneSecretKey.java | 0 .../hadoop/ozone/security/OzoneSecretManager.java | 0 .../hadoop/ozone/security/OzoneSecretStore.java | 0 .../hadoop/ozone/security}/package-info.java | 7 +- .../web/ozShell/bucket/AddAclBucketHandler.java | 105 --- .../web/ozShell/bucket/CreateBucketHandler.java | 107 --- .../web/ozShell/bucket/DeleteBucketHandler.java | 64 -- .../web/ozShell/bucket/GetAclBucketHandler.java | 85 --- .../web/ozShell/bucket/InfoBucketHandler.java | 68 -- .../web/ozShell/bucket/ListBucketHandler.java | 105 --- .../web/ozShell/bucket/RemoveAclBucketHandler.java | 106 --- .../web/ozShell/bucket/SetAclBucketHandler.java | 100 --- .../ozone/web/ozShell/keys/AddAclKeyHandler.java | 109 --- .../ozone/web/ozShell/keys/DeleteKeyHandler.java | 70 -- .../ozone/web/ozShell/keys/GetAclKeyHandler.java | 89 --- .../ozone/web/ozShell/keys/GetKeyHandler.java | 113 --- .../ozone/web/ozShell/keys/InfoKeyHandler.java | 78 -- .../ozone/web/ozShell/keys/ListKeyHandler.java | 113 --- .../ozone/web/ozShell/keys/PutKeyHandler.java | 129 ---- .../web/ozShell/keys/RemoveAclKeyHandler.java | 109 --- .../ozone/web/ozShell/keys/SetAclKeyHandler.java | 102 --- .../ozone/web/ozShell/s3/GetS3SecretHandler.java | 63 -- .../web/ozShell/token/CancelTokenHandler.java | 76 -- .../ozone/web/ozShell/token/GetTokenHandler.java | 81 -- .../ozone/web/ozShell/token/PrintTokenHandler.java | 71 -- .../ozone/web/ozShell/token/RenewTokenHandler.java | 79 -- .../web/ozShell/volume/AddAclVolumeHandler.java | 103 --- .../web/ozShell/volume/CreateVolumeHandler.java | 103 --- .../web/ozShell/volume/DeleteVolumeHandler.java | 62 -- .../web/ozShell/volume/GetAclVolumeHandler.java | 80 -- .../web/ozShell/volume/ListVolumeHandler.java | 111 --- .../web/ozShell/volume/RemoveAclVolumeHandler.java | 103 --- .../web/ozShell/volume/SetAclVolumeHandler.java | 106 --- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 13 +- .../org/apache/hadoop/ozone/om/TestOMStorage.java | 9 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 26 +- .../ozone/om/TestOzoneManagerHttpServer.java | 21 +- .../om/codec/TestOmMultipartKeyInfoCodec.java | 0 .../ozone/om/codec/TestOmPrefixInfoCodec.java | 0 .../ozone/om/codec/TestS3SecretValueCodec.java | 0 .../apache/hadoop/ozone/om/codec/package-info.java | 0 .../request/bucket/TestOMBucketCreateRequest.java | 5 + .../om/request/key/TestOMKeyCommitRequest.java | 42 ++ .../s3/bucket/TestS3BucketCreateRequest.java | 4 +- .../request/volume/TestOMVolumeCreateRequest.java | 5 + .../volume/TestOMVolumeSetOwnerRequest.java | 49 ++ .../om/response/key/TestOMKeyCommitResponse.java | 14 +- .../ozone/security/TestAWSV4AuthValidator.java | 0 .../TestOzoneDelegationTokenSecretManager.java | 14 +- .../ozone/security/TestOzoneTokenIdentifier.java | 41 +- hadoop-ozone/ozonefs-lib-current/pom.xml | 17 +- hadoop-ozone/ozonefs-lib-legacy/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 36 +- .../java/org/apache/hadoop/fs/ozone/BasicOzFs.java | 12 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 13 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 20 +- .../org/apache/hadoop/fs/ozone/O3fsDtFetcher.java | 8 +- .../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 12 +- .../hadoop/fs/ozone/OzoneClientAdapterImpl.java | 5 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 10 +- .../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 5 +- .../fs/ozone/TestOzoneFileSystemWithMocks.java | 31 +- .../hadoop/fs/ozone/TestReadWriteStatistics.java | 41 +- hadoop-ozone/pom.xml | 29 +- hadoop-ozone/recon-codegen/pom.xml | 7 +- .../ozone/recon/codegen/JooqCodeGenerator.java | 2 +- .../recon/codegen/ReconSchemaGenerationModule.java | 2 + .../recon/schema/ContainerSchemaDefinition.java | 87 +++ .../recon/schema/ReconTaskSchemaDefinition.java | 2 + .../ozone/recon/schema/StatsSchemaDefinition.java | 2 + .../recon/schema/UtilizationSchemaDefinition.java | 19 +- hadoop-ozone/recon/pom.xml | 33 +- .../hadoop/ozone/recon/ConfigurationProvider.java | 5 +- .../hadoop/ozone/recon/ReconControllerModule.java | 51 +- .../hadoop/ozone/recon/ReconSchemaManager.java | 4 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 10 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 54 +- .../ozone/recon/api/ClusterStateEndpoint.java | 36 +- .../hadoop/ozone/recon/api/ContainerEndpoint.java | 29 +- .../hadoop/ozone/recon/api/types/KeysResponse.java | 61 +- .../recon/api/types/MissingContainerMetadata.java | 14 +- .../ozone/recon/fsck/MissingContainerTask.java | 22 +- .../recon/metrics/OzoneManagerSyncMetrics.java | 133 ++++ .../hadoop/ozone/recon/metrics}/package-info.java | 16 +- .../recon/persistence/ContainerSchemaManager.java | 104 +++ .../recon/recovery/ReconOMMetadataManager.java | 6 + .../recon/recovery/ReconOmMetadataManagerImpl.java | 10 + .../ozone/recon/scm/ReconContainerManager.java | 55 +- .../hadoop/ozone/recon/scm/ReconDBDefinition.java} | 31 +- .../recon/scm/ReconDatanodeProtocolServer.java | 4 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 17 +- .../ozone/recon/scm/ReconPipelineFactory.java | 4 +- .../ozone/recon/scm/ReconPipelineManager.java | 34 +- .../recon/scm/ReconPipelineReportHandler.java | 4 +- .../scm/ReconStorageContainerManagerFacade.java | 46 +- .../spi/impl/ContainerDBServiceProviderImpl.java | 10 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 76 +- .../webapps/recon/ozone-recon-web/api/db.json | 74 +- .../webapps/recon/ozone-recon-web/api/routes.json | 3 +- .../recon/ozone-recon-web/src/utils/common.tsx | 9 +- .../views/MissingContainers/MissingContainers.less | 4 + .../views/MissingContainers/MissingContainers.tsx | 99 ++- .../src/views/Overview/Overview.tsx | 12 +- ...erTest.java => OMMetadataManagerTestUtils.java} | 60 +- .../hadoop/ozone/recon/ReconTestInjector.java | 337 +++++++++ .../apache/hadoop/ozone/recon/TestReconUtils.java | 45 +- .../ozone/recon/api/TestContainerEndpoint.java | 170 ++--- .../hadoop/ozone/recon/api/TestEndpoints.java | 141 ++-- .../ozone/recon/api/TestTaskStatusService.java | 23 +- .../ozone/recon/fsck/TestMissingContainerTask.java | 40 +- ...tabaseTest.java => AbstractReconSqlDBTest.java} | 131 +++- .../TestReconInternalSchemaDefinition.java | 26 +- .../persistence/TestStatsSchemaDefinition.java | 25 +- .../TestUtilizationSchemaDefinition.java | 37 +- .../scm/AbstractReconContainerManagerTest.java | 38 +- .../ozone/recon/scm/TestReconContainerManager.java | 2 +- .../ozone/recon/scm/TestReconPipelineManager.java | 37 +- .../recon/scm/TestReconPipelineReportHandler.java | 2 +- .../impl/TestContainerDBServiceProviderImpl.java | 56 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 64 +- .../recon/tasks/TestContainerKeyMapperTask.java | 71 +- .../ozone/recon/tasks/TestFileSizeCountTask.java | 16 +- .../recon/tasks/TestReconTaskControllerImpl.java | 28 +- .../recon/types/GuiceInjectorUtilsForTests.java | 136 ---- hadoop-ozone/s3gateway/pom.xml | 36 +- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 2 + .../hadoop/ozone/s3/S3GatewayHttpServer.java | 4 +- .../org/apache/hadoop/ozone/s3/TracingFilter.java | 82 +++ hadoop-ozone/tools/pom.xml | 31 +- .../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 58 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 12 +- .../hadoop/ozone/debug/ChunkDataNodeDetails.java} | 31 +- .../apache/hadoop/ozone/debug/ChunkDetails.java} | 44 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 171 +++++ .../hadoop/ozone/debug/ContainerChunkInfo.java | 93 +++ .../org/apache/hadoop/ozone/debug/OzoneDebug.java | 49 +- .../apache/hadoop/ozone/debug}/package-info.java | 7 +- .../hadoop/ozone/freon/BaseAppendLogGenerator.java | 10 + .../hadoop/ozone/freon/BaseFreonGenerator.java | 17 +- .../freon/FollowerAppendLogEntryGenerator.java | 8 - .../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +- .../apache/hadoop/ozone/freon/FreonHttpServer.java | 4 +- .../ozone/freon/HadoopNestedDirGenerator.java | 113 +++ .../ozone/freon/LeaderAppendLogEntryGenerator.java | 273 +++++++ .../hadoop/ozone/freon/RandomKeyGenerator.java | 27 +- .../ozone/genesis/BenchMarkContainerStateMap.java | 8 +- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 5 +- .../ozone/genesis/BenchMarkOzoneManager.java | 11 +- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 11 +- .../ozone/genesis/BenchmarkChunkManager.java | 33 +- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 58 +- .../org/apache/hadoop/ozone/shell/Handler.java | 133 ++++ .../org/apache/hadoop/ozone/shell/ListOptions.java | 58 ++ .../apache/hadoop/ozone/shell}/OzoneAddress.java | 45 +- .../org/apache/hadoop/ozone/shell}/OzoneShell.java | 22 +- .../java/org/apache/hadoop/ozone/shell}/Shell.java | 27 +- .../hadoop/ozone/shell/StoreTypeOption.java} | 36 +- .../apache/hadoop/ozone/shell/acl/AclHandler.java | 62 ++ .../apache/hadoop/ozone/shell/acl/AclOption.java | 91 +++ .../hadoop/ozone/shell/acl/GetAclHandler.java} | 24 +- .../hadoop/ozone/shell/acl}/package-info.java | 4 +- .../ozone/shell/bucket/AddAclBucketHandler.java} | 36 +- .../hadoop/ozone/shell}/bucket/BucketCommands.java | 4 +- .../hadoop/ozone/shell/bucket/BucketHandler.java} | 23 +- .../hadoop/ozone/shell/bucket/BucketUri.java | 44 +- .../ozone/shell/bucket/CreateBucketHandler.java | 88 +++ .../ozone/shell/bucket/DeleteBucketHandler.java} | 33 +- .../ozone/shell/bucket/GetAclBucketHandler.java} | 28 +- .../ozone/shell/bucket/InfoBucketHandler.java} | 32 +- .../ozone/shell/bucket/ListBucketHandler.java | 68 ++ .../shell/bucket/RemoveAclBucketHandler.java} | 36 +- .../ozone/shell/bucket/SetAclBucketHandler.java} | 37 +- .../hadoop/ozone/shell/bucket}/package-info.java | 4 +- .../hadoop/ozone/shell/keys/AddAclKeyHandler.java} | 36 +- .../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 50 ++ .../hadoop/ozone/shell/keys/GetAclKeyHandler.java} | 28 +- .../hadoop/ozone/shell/keys/GetKeyHandler.java | 98 +++ .../hadoop/ozone/shell/keys/InfoKeyHandler.java | 58 ++ .../hadoop/ozone/shell}/keys/KeyCommands.java | 4 +- .../hadoop/ozone/shell/keys/KeyHandler.java} | 23 +- .../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 44 +- .../hadoop/ozone/shell/keys/ListKeyHandler.java | 78 ++ .../hadoop/ozone/shell/keys/PutKeyHandler.java | 108 +++ .../ozone/shell/keys/RemoveAclKeyHandler.java} | 36 +- .../hadoop/ozone/shell}/keys/RenameKeyHandler.java | 45 +- .../hadoop/ozone/shell/keys/SetAclKeyHandler.java} | 37 +- .../hadoop/ozone/shell/keys}/package-info.java | 4 +- .../apache/hadoop/ozone/shell}/package-info.java | 7 +- .../hadoop/ozone/shell/s3/GetS3SecretHandler.java} | 30 +- .../hadoop/ozone/shell}/s3/S3BucketMapping.java | 42 +- .../apache/hadoop/ozone/shell}/s3/S3Handler.java | 30 +- .../org/apache/hadoop/ozone/shell}/s3/S3Shell.java | 17 +- .../hadoop/ozone/shell}/s3/package-info.java | 2 +- .../ozone/shell/token/CancelTokenHandler.java} | 29 +- .../hadoop/ozone/shell/token/GetTokenHandler.java | 72 ++ .../ozone/shell/token/PrintTokenHandler.java} | 46 +- .../ozone/shell/token/RenewTokenHandler.java} | 29 +- .../hadoop/ozone/shell/token/RenewerOption.java} | 32 +- .../hadoop/ozone/shell}/token/TokenCommands.java | 5 +- .../hadoop/ozone/shell/token/TokenHandler.java | 54 ++ .../hadoop/ozone/shell/token/TokenOption.java | 56 ++ .../hadoop/ozone/shell}/token/package-info.java | 6 +- .../ozone/shell/volume/AddAclVolumeHandler.java} | 36 +- .../ozone/shell/volume/CreateVolumeHandler.java | 74 ++ .../ozone/shell/volume/DeleteVolumeHandler.java} | 30 +- .../ozone/shell/volume/GetAclVolumeHandler.java} | 28 +- .../ozone/shell}/volume/InfoVolumeHandler.java | 36 +- .../ozone/shell/volume/ListVolumeHandler.java | 99 +++ .../shell/volume/RemoveAclVolumeHandler.java} | 36 +- .../ozone/shell/volume/SetAclVolumeHandler.java} | 37 +- .../ozone/shell}/volume/UpdateVolumeHandler.java | 46 +- .../hadoop/ozone/shell}/volume/VolumeCommands.java | 4 +- .../hadoop/ozone/shell/volume/VolumeHandler.java} | 23 +- .../hadoop/ozone/shell/volume/VolumeUri.java | 44 +- .../hadoop/ozone/shell/volume}/package-info.java | 4 +- .../hadoop/ozone/shell}/TestOzoneAddress.java | 2 +- .../apache/hadoop/ozone/shell}/package-info.java | 2 +- hadoop-ozone/upgrade/pom.xml | 9 +- pom.xml | 47 +- 763 files changed, 19402 insertions(+), 11091 deletions(-) diff --cc hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config index 32a9a94,6184b42..ae1f558 --- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config +++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config @@@ -14,8 -14,8 +14,9 @@@ # See the License for the specific language governing permissions and # limitations under the License. +CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem + OZONE-SITE.XML_ozone.om.volume.listall.allowed=false OZONE-SITE.XML_ozone.om.address=om OZONE-SITE.XML_ozone.om.http-address=om:9874 diff --cc hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot index adc6e0f,fcc7809..e6ab0d3 --- a/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot +++ b/hadoop-ozone/dist/src/main/smoketest/ozonefs/ozonefs.robot @@@ -14,51 -14,35 +14,56 @@@ # limitations under the License. *** Settings *** -Documentation Ozonefs test +Documentation Ozonefs test covering both o3fs and ofs Library OperatingSystem Resource ../commonlib.robot + Test Timeout 5 minutes *** Variables *** +${OfsBucket1} om:9862/fstest/bucket1 +${O3fsBucket1} bucket1.fstest/ +${O3Bucket1} o3://om/fstest/bucket1 +${O3Bucket2} o3://om/fstest/bucket2 +${O3Bucket3} o3://om/fstest2/bucket3 +${OfsNonExistBucket} om:9862/abc/def +${O3fsNonExistBucket} def.abc/ +${NonExistVolume} abc *** Test Cases *** -Create volume and bucket +Create volume and bucket for Ozone file System test Execute ozone sh volume create o3://om/fstest --quota 100TB + Execute ozone sh bucket create ${O3Bucket1} + Execute ozone sh bucket create ${O3Bucket2} + Execute ozone sh volume create o3://om/fstest2 --quota 100TB - Execute ozone sh bucket create o3://om/fstest/bucket1 - Execute ozone sh bucket create o3://om/fstest/bucket2 - Execute ozone sh bucket create o3://om/fstest2/bucket3 + Execute ozone sh bucket create ${O3Bucket3} + -Check volume from ozonefs +Check volume from o3fs + ${result} = Execute ozone sh volume list + Should contain ${result} fstest + Should contain ${result} fstest2 + Should Match Regexp ${result} "admin" : "(hadoop|testuser\/scm@EXAMPLE\.COM)" - ${result} = Execute ozone fs -ls o3fs://bucket1.fstest/ + ${result} = Execute ozone fs -ls o3fs://${O3fsBucket1} + +Test ozone shell with ofs + Test ozone shell with scheme ofs om:9862/fstest/bucket1 om:9862/fstest/bucket2 om:9862/fstest2/bucket3 om:9862/abc/def -Run ozoneFS tests - Execute ozone fs -mkdir -p o3fs://bucket1.fstest/testdir/deep - ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name' +Test ozone shell with o3fs + Test ozone shell with scheme o3fs bucket1.fstest/ bucket2.fstest/ bucket3.fstest2/ def.abc/ + +*** Keywords *** +Test ozone shell with scheme + [arguments] ${scheme} ${testBucket1} ${testBucket2} ${testBucket3} ${nonExistBucket} + + ${result} = Execute ozone fs -ls ${scheme}://${testBucket1} + Execute ozone fs -mkdir -p ${scheme}://${testBucket1}/testdir/deep + ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name' Should contain ${result} testdir/deep - Execute ozone fs -copyFromLocal NOTICE.txt o3fs://bucket1.fstest/testdir/deep/ - ${result} = Execute ozone sh key list o3://om/fstest/bucket1 | jq -r '.name' + + Execute ozone fs -copyFromLocal NOTICE.txt ${scheme}://${testBucket1}/testdir/deep/ + ${result} = Execute ozone sh key list ${O3Bucket1} | jq -r '.name' Should contain ${result} NOTICE.txt ${result} = Execute ozone sh key info o3://om/fstest/bucket1/testdir/deep/NOTICE.txt | jq -r '.replicationFactor' Should Be Equal ${result} 3 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
