This is an automated email from the ASF dual-hosted git repository. erose pushed a commit to branch HDDS-10239-container-reconciliation in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2cfd22a407c6cd8e5b06c4503390bfbfbea7c702 Merge: cfcabf1422 98c765402e Author: Ethan Rose <[email protected]> AuthorDate: Wed Feb 5 18:12:28 2025 -0500 Merge branch 'master' into HDDS-10239-container-reconciliation * master: (168 commits) HDDS-12112. Fix interval used for Chunk Read/Write Dashboard (#7724) HDDS-12212. Fix grammar in decommissioning and observability documentation (#7815) HDDS-12195. Implement skip() in OzoneFSInputStream (#7801) HDDS-12200. Fix grammar in OM HA, EC and Snapshot doc (#7806) HDDS-12202. OpsCreate and OpsAppend metrics not incremented (#7811) HDDS-12203. Initialize block length before skip (#7809) HDDS-12183. Reuse cluster across safe test classes (#7793) HDDS-11714. resetDeletedBlockRetryCount with --all may fail and can cause long db lock in large cluster. (#7665) HDDS-12186. (addendum) Avoid array allocation for table iterator (#7799) HDDS-12186. Avoid array allocation for table iterator. (#7797) HDDS-11508. Decouple delete batch limits from Ratis request size for DirectoryDeletingService. (#7365) HDDS-12073. Don't show Source Bucket and Volume if null in DU metadata (#7760) HDDS-12142. Save logs from build check (#7782) HDDS-12163. Reduce number of individual getCapacity/getAvailable/getUsedSpace calls (#7790) HDDS-12176. Trivial dependency cleanup.(#7787) HDDS-12181. Bump jline to 3.29.0 (#7789) HDDS-12165. Refactor VolumeInfoMetrics to use getCurrentUsage (#7784) HDDS-12085. Add manual refresh button for DU page (#7780) HDDS-12132. Parameterize testUpdateTransactionInfoTable for SCM (#7768) HDDS-11277. Remove dependency on hadoop-hdfs in Ozone client (#7781) ... Conflicts: hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java hadoop-ozone/dist/src/main/smoketest/admincli/container.robot hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java .github/workflows/ci.yml | 48 +- .github/workflows/intermittent-test-check.yml | 2 +- .github/workflows/repeat-acceptance.yml | 2 +- .mvn/develocity.xml | 3 +- .mvn/extensions.xml | 2 +- dev-support/ci/integration_suites.sh | 27 + dev-support/ci/selective_ci_checks.sh | 6 +- hadoop-hdds/annotations/pom.xml | 15 +- hadoop-hdds/client/pom.xml | 60 +- .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 12 +- .../hdds/scm/storage/BlockDataStreamOutput.java | 25 +- .../hdds/scm/storage/ByteBufferStreamOutput.java | 4 +- .../hdds/scm/storage/MultipartInputStream.java | 5 + .../hadoop/hdds/scm/storage/StreamBuffer.java | 4 +- hadoop-hdds/common/pom.xml | 215 ++-- .../org/apache/hadoop/hdds/ComponentVersion.java | 9 +- .../org/apache/hadoop/hdds/cli/AbstractMixin.java | 41 +- .../apache/hadoop/hdds/cli/AbstractSubcommand.java | 89 ++ .../hadoop/hdds/cli/ExtensibleParentCommand.java | 6 +- .../org/apache/hadoop/hdds/cli/GenericCli.java | 75 +- .../hadoop/hdds/cli/GenericParentCommand.java | 3 +- .../apache/hadoop/hdds/client/ReplicationType.java | 1 + .../hadoop/hdds/conf/DefaultConfigManager.java | 58 - .../apache/hadoop/hdds/fs/SpaceUsageSource.java | 5 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 11 +- .../balancer/ContainerBalancerConfiguration.java | 0 .../hdds/scm/container/balancer/package-info.java | 15 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 12 +- .../ContainerCommandResponseBuilders.java | 4 +- .../hadoop/hdds/security/SecurityConfig.java | 33 +- .../hadoop/hdds/security/SecurityConstants.java | 49 + .../hadoop/hdds/security/x509/keys/KeyCodec.java | 128 +++ .../hadoop/hdds/security/x509/keys/KeyStorage.java | 249 +++++ .../org/apache/hadoop/ozone/ClientVersion.java | 5 +- .../apache/hadoop/ozone/OzoneManagerVersion.java | 3 + .../java/org/apache/hadoop/ozone/Versioned.java | 11 +- .../apache/hadoop/ozone/common/ChecksumCache.java | 2 +- .../apache/hadoop/ozone/common/ChunkBuffer.java | 55 +- .../ozone/common/ChunkBufferToByteString.java | 93 ++ .../common/ChunkBufferToByteStringByByteBufs.java | 101 ++ .../ozone/common/IncrementalChunkBuffer.java | 4 +- .../apache/hadoop/ozone/upgrade/LayoutFeature.java | 9 +- .../common/src/main/resources/ozone-default.xml | 72 +- .../security/x509/keys/TestHDDSKeyGenerator.java | 0 .../hdds/security/x509/keys/TestKeyCodec.java | 78 ++ .../hdds/security/x509/keys/TestKeyStorage.java | 344 ++++++ .../hdds/security/x509/keys/package-info.java | 0 hadoop-hdds/config/pom.xml | 12 +- hadoop-hdds/container-service/pom.xml | 161 ++- .../apache/hadoop/ozone/HddsDatanodeService.java | 25 +- .../common/impl/ContainerLayoutVersion.java | 3 +- .../container/common/impl/HddsDispatcher.java | 7 +- .../common/statemachine/DatanodeConfiguration.java | 14 + .../transport/server/GrpcXceiverService.java | 13 +- .../server/ratis/ContainerStateMachine.java | 40 +- .../transport/server/ratis/DispatcherContext.java | 30 +- .../common/volume/AvailableSpaceFilter.java | 7 +- .../volume/CapacityVolumeChoosingPolicy.java | 4 +- .../ozone/container/common/volume/DbVolume.java | 4 +- .../ozone/container/common/volume/HddsVolume.java | 7 +- .../container/common/volume/MutableVolumeSet.java | 8 +- .../container/common/volume/StorageVolume.java | 14 - .../ozone/container/common/volume/VolumeInfo.java | 22 - .../container/common/volume/VolumeInfoMetrics.java | 111 +- .../ozone/container/common/volume/VolumeUsage.java | 24 +- .../ozone/container/keyvalue/KeyValueHandler.java | 56 +- .../container/keyvalue/helpers/ChunkUtils.java | 40 +- .../container/keyvalue/impl/BlockManagerImpl.java | 12 +- .../keyvalue/impl/ChunkManagerDispatcher.java | 7 +- .../keyvalue/impl/FilePerBlockStrategy.java | 11 +- .../keyvalue/impl/FilePerChunkStrategy.java | 10 +- .../keyvalue/impl/KeyValueStreamDataChannel.java | 65 +- .../keyvalue/interfaces/BlockManager.java | 3 + .../keyvalue/interfaces/ChunkManager.java | 3 +- .../replication/ReplicationSupervisor.java | 14 +- .../upgrade/VersionedDatanodeFeatures.java | 6 +- .../hadoop/ozone/TestHddsSecureDatanodeInit.java | 22 +- .../common/impl/TestContainerPersistence.java | 6 +- .../volume/TestCapacityVolumeChoosingPolicy.java | 6 +- .../container/common/volume/TestHddsVolume.java | 40 +- .../common/volume/TestReservedVolumeSpace.java | 4 +- .../volume/TestRoundRobinVolumeChoosingPolicy.java | 4 +- .../container/common/volume/TestVolumeSet.java | 2 +- .../keyvalue/ContainerLayoutTestInfo.java | 2 +- .../container/keyvalue/TestKeyValueHandler.java | 75 +- .../keyvalue/impl/TestChunkManagerDummyImpl.java | 4 +- .../keyvalue/impl/TestFilePerBlockStrategy.java | 15 +- .../impl/TestKeyValueStreamDataChannel.java | 65 +- .../container/ozoneimpl/TestOzoneContainer.java | 2 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 604 ----------- hadoop-hdds/crypto-api/pom.xml | 35 +- hadoop-hdds/crypto-default/pom.xml | 35 +- hadoop-hdds/docs/content/feature/Decommission.md | 2 +- hadoop-hdds/docs/content/feature/ErasureCoding.md | 6 +- hadoop-hdds/docs/content/feature/OM-HA.md | 10 +- hadoop-hdds/docs/content/feature/OM-HA.zh.md | 9 - hadoop-hdds/docs/content/feature/Observability.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 24 +- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 27 +- hadoop-hdds/docs/content/feature/Snapshot.md | 4 +- hadoop-hdds/docs/content/integration/DistCp.md | 89 ++ hadoop-hdds/docs/content/integration/Hive.md | 169 +++ hadoop-hdds/docs/content/integration/Impala.md | 101 ++ hadoop-hdds/docs/content/integration/_index.md | 26 + hadoop-hdds/docs/pom.xml | 12 +- .../static/swagger-resources/recon-api.yaml | 1146 ++++++++++++++------ hadoop-hdds/erasurecode/pom.xml | 29 +- hadoop-hdds/framework/pom.xml | 275 +++-- .../apache/hadoop/hdds/conf/HddsConfServlet.java | 9 - .../hadoop/hdds/security/OzoneSecretKey.java | 23 +- .../certificate/authority/DefaultCAServer.java | 110 +- .../client/DefaultCertificateClient.java | 50 +- .../hadoop/hdds/security/x509/keys/KeyCodec.java | 425 -------- .../hdds/security/x509/keys/SecurityUtil.java | 90 -- .../hadoop/hdds/server/http/BaseHttpServer.java | 22 +- .../hadoop/hdds/server/http/HttpServer2.java | 47 +- .../org/apache/hadoop/hdds/utils/db/Table.java | 47 + .../apache/hadoop/hdds/utils/db/TypedTable.java | 11 +- .../certificate/authority/TestDefaultCAServer.java | 57 +- .../client/TestDefaultCertificateClient.java | 21 +- .../client/TestDnCertificateClientInit.java | 10 +- .../hdds/security/x509/keys/TestKeyCodec.java | 235 ---- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 5 +- hadoop-hdds/hadoop-dependency-client/pom.xml | 251 ++--- hadoop-hdds/hadoop-dependency-server/pom.xml | 125 +-- hadoop-hdds/hadoop-dependency-test/pom.xml | 35 +- hadoop-hdds/interface-admin/pom.xml | 20 +- hadoop-hdds/interface-client/pom.xml | 80 +- hadoop-hdds/interface-server/pom.xml | 60 +- hadoop-hdds/managed-rocksdb/pom.xml | 31 +- hadoop-hdds/pom.xml | 130 +-- hadoop-hdds/rocks-native/pom.xml | 125 +-- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 69 +- hadoop-hdds/server-scm/pom.xml | 184 ++-- .../hadoop/hdds/scm/block/DeletedBlockLog.java | 4 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 46 +- .../hdds/scm/container/ContainerManager.java | 3 + .../hdds/scm/container/ContainerManagerImpl.java | 7 + .../hdds/scm/container/ContainerStateManager.java | 7 + .../scm/container/ContainerStateManagerImpl.java | 8 + .../AbstractOverReplicationHandler.java | 5 +- .../container/replication/ContainerReplicaOp.java | 12 +- .../replication/ContainerReplicaPendingOps.java | 36 +- .../replication/ECUnderReplicationHandler.java | 2 +- .../replication/RatisOverReplicationHandler.java | 12 +- .../container/replication/ReplicationManager.java | 41 +- .../scm/container/states/ContainerStateMap.java | 2 +- .../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 33 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 171 +-- .../hdds/scm/safemode/DataNodeSafeModeRule.java | 6 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 7 +- .../safemode/OneReplicaPipelineSafeModeRule.java | 6 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 5 - .../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 12 + .../hdds/scm/safemode/SafeModeRuleFactory.java | 35 +- .../hdds/scm/security/RootCARotationManager.java | 9 +- .../hdds/scm/server/StorageContainerManager.java | 14 +- .../scm/server/StorageContainerManagerStarter.java | 9 +- .../ScmHAUnfinalizedStateValidationAction.java | 6 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 3 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 - .../scm/container/balancer/TestMoveManager.java | 28 +- .../TestContainerReplicaPendingOps.java | 176 +-- .../replication/TestECContainerReplicaCount.java | 4 +- .../TestRatisOverReplicationHandler.java | 44 +- .../replication/TestReplicationManager.java | 38 +- .../TestReplicationManagerScenarios.java | 4 +- ...onfiguration.java => TestSCMConfiguration.java} | 83 +- .../ha/TestStatefulServiceStateManagerImpl.java | 2 - .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 5 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 4 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 6 +- .../TestOneReplicaPipelineSafeModeRule.java | 2 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +- .../TestSCMHAUnfinalizedStateValidationAction.java | 57 +- hadoop-hdds/test-utils/pom.xml | 47 +- .../org/apache/ozone/test/GenericTestUtils.java | 127 ++- .../main/java/org/apache/ozone/test/IntLambda.java | 33 +- hadoop-hdds/tools/pom.xml | 129 +-- .../hdds/scm/cli/ContainerBalancerCommands.java | 15 +- .../hdds/scm/cli/ReplicationManagerCommands.java | 15 +- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 15 +- .../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 17 +- .../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 3 +- .../hadoop/hdds/scm/cli/cert/CertCommands.java | 16 +- .../hdds/scm/cli/container/ContainerCommands.java | 24 +- .../hdds/scm/cli/container/InfoSubcommand.java | 11 +- .../hdds/scm/cli/container/ListSubcommand.java | 6 +- .../hdds/scm/cli/container/UpgradeSubcommand.java | 15 +- .../hdds/scm/cli/datanode/DatanodeCommands.java | 15 +- .../scm/cli/datanode/DecommissionSubCommand.java | 54 +- .../hdds/scm/cli/datanode/HostNameParameters.java | 53 + .../scm/cli/datanode/MaintenanceSubCommand.java | 50 +- .../scm/cli/datanode/RecommissionSubCommand.java | 48 +- .../hdds/scm/cli/datanode/StatusSubCommand.java | 13 +- .../hdds/scm/cli/pipeline/PipelineCommands.java | 15 +- .../dev-support/findbugsExcludeFile.xml} | 13 +- .../tools => hadoop-ozone/cli-shell}/pom.xml | 152 ++- .../hadoop/ozone/shell/ClearSpaceQuotaOptions.java | 0 .../org/apache/hadoop/ozone/shell/Handler.java | 39 +- .../org/apache/hadoop/ozone/shell/ListOptions.java | 0 .../ozone/shell/MandatoryReplicationOptions.java | 0 .../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +- .../org/apache/hadoop/ozone/shell/OzoneShell.java | 15 - .../java/org/apache/hadoop/ozone/shell/REPL.java | 17 +- .../hadoop/ozone/shell/ReplicationOptions.java | 0 .../hadoop/ozone/shell/SetSpaceQuotaOptions.java | 0 .../java/org/apache/hadoop/ozone/shell/Shell.java | 41 +- .../ozone/shell/ShellReplicationOptions.java | 0 .../apache/hadoop/ozone/shell/StoreTypeOption.java | 0 .../apache/hadoop/ozone/shell/acl/AclHandler.java | 0 .../apache/hadoop/ozone/shell/acl/AclOption.java | 8 +- .../hadoop/ozone/shell/acl/GetAclHandler.java | 0 .../hadoop/ozone/shell/acl/package-info.java | 0 .../ozone/shell/bucket/AddAclBucketHandler.java | 0 .../hadoop/ozone/shell/bucket/BucketCommands.java | 28 +- .../hadoop/ozone/shell/bucket/BucketHandler.java | 0 .../hadoop/ozone/shell/bucket/BucketUri.java | 0 .../ozone/shell/bucket/ClearQuotaHandler.java | 0 .../ozone/shell/bucket/CreateBucketHandler.java | 33 +- .../ozone/shell/bucket/DeleteBucketHandler.java | 0 .../ozone/shell/bucket/GetAclBucketHandler.java | 0 .../ozone/shell/bucket/InfoBucketHandler.java | 0 .../ozone/shell/bucket/LinkBucketHandler.java | 0 .../ozone/shell/bucket/ListBucketHandler.java | 0 .../ozone/shell/bucket/RemoveAclBucketHandler.java | 0 .../ozone/shell/bucket/SetAclBucketHandler.java | 0 .../ozone/shell/bucket/SetEncryptionKey.java | 0 .../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 0 .../shell/bucket/SetReplicationConfigHandler.java | 0 .../ozone/shell/bucket/UpdateBucketHandler.java | 0 .../hadoop/ozone/shell/bucket/package-info.java | 0 .../ozone/shell/common/VolumeBucketHandler.java | 0 .../hadoop/ozone/shell/common/VolumeBucketUri.java | 0 .../hadoop/ozone/shell/common/package-info.java | 0 .../hadoop/ozone/shell/keys/AddAclKeyHandler.java | 0 .../hadoop/ozone/shell/keys/CatKeyHandler.java | 0 .../ozone/shell/keys/ChecksumKeyHandler.java | 0 .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 0 .../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 0 .../hadoop/ozone/shell/keys/GetAclKeyHandler.java | 0 .../hadoop/ozone/shell/keys/GetKeyHandler.java | 0 .../hadoop/ozone/shell/keys/InfoKeyHandler.java | 0 .../hadoop/ozone/shell/keys/KeyCommands.java | 28 +- .../apache/hadoop/ozone/shell/keys/KeyHandler.java | 0 .../org/apache/hadoop/ozone/shell/keys/KeyUri.java | 0 .../hadoop/ozone/shell/keys/ListKeyHandler.java | 0 .../hadoop/ozone/shell/keys/PutKeyHandler.java | 0 .../ozone/shell/keys/RemoveAclKeyHandler.java | 0 .../hadoop/ozone/shell/keys/RenameKeyHandler.java | 0 .../hadoop/ozone/shell/keys/RewriteKeyHandler.java | 0 .../hadoop/ozone/shell/keys/SetAclKeyHandler.java | 0 .../hadoop/ozone/shell/keys/package-info.java | 0 .../apache/hadoop/ozone/shell}/package-info.java | 4 +- .../ozone/shell/prefix/AddAclPrefixHandler.java | 0 .../ozone/shell/prefix/GetAclPrefixHandler.java | 0 .../hadoop/ozone/shell/prefix/PrefixCommands.java | 27 +- .../hadoop/ozone/shell/prefix/PrefixUri.java | 0 .../ozone/shell/prefix/RemoveAclPrefixHandler.java | 0 .../ozone/shell/prefix/SetAclPrefixHandler.java | 0 .../hadoop/ozone/shell/prefix/package-info.java | 0 .../hadoop/ozone/shell/s3/GetS3SecretHandler.java | 0 .../ozone/shell/s3/RevokeS3SecretHandler.java | 0 .../apache/hadoop/ozone/shell/s3/S3Handler.java | 0 .../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 14 - .../hadoop/ozone/shell/s3/SetS3SecretHandler.java | 0 .../apache/hadoop/ozone/shell/s3/package-info.java | 0 .../shell/snapshot/BucketSnapshotHandler.java | 0 .../shell/snapshot/CreateSnapshotHandler.java | 0 .../shell/snapshot/DeleteSnapshotHandler.java | 0 .../ozone/shell/snapshot/InfoSnapshotHandler.java | 0 .../shell/snapshot/ListSnapshotDiffHandler.java | 0 .../ozone/shell/snapshot/ListSnapshotHandler.java | 0 .../shell/snapshot/RenameSnapshotHandler.java | 0 .../ozone/shell/snapshot/SnapshotCommands.java | 27 +- .../ozone/shell/snapshot/SnapshotDiffHandler.java | 12 +- .../hadoop/ozone/shell/snapshot/SnapshotUri.java | 0 .../hadoop/ozone/shell/snapshot/package-info.java | 0 .../ozone/shell/tenant/GetUserInfoHandler.java | 10 - .../shell/tenant/TenantAssignAdminHandler.java | 0 .../tenant/TenantAssignUserAccessIdHandler.java | 0 .../shell/tenant/TenantBucketLinkHandler.java | 0 .../ozone/shell/tenant/TenantCreateHandler.java | 0 .../ozone/shell/tenant/TenantDeleteHandler.java | 0 .../ozone/shell/tenant/TenantGetSecretHandler.java | 0 .../hadoop/ozone/shell/tenant/TenantHandler.java | 0 .../ozone/shell/tenant/TenantListHandler.java | 0 .../ozone/shell/tenant/TenantListUsersHandler.java | 0 .../shell/tenant/TenantRevokeAdminHandler.java | 0 .../tenant/TenantRevokeUserAccessIdHandler.java | 0 .../ozone/shell/tenant/TenantSetSecretHandler.java | 0 .../hadoop/ozone/shell/tenant/TenantShell.java | 15 +- .../ozone/shell/tenant/TenantUserCommands.java | 27 +- .../hadoop/ozone/shell/tenant/package-info.java | 0 .../ozone/shell/token/CancelTokenHandler.java | 0 .../hadoop/ozone/shell/token/GetTokenHandler.java | 0 .../ozone/shell/token/PrintTokenHandler.java | 0 .../ozone/shell/token/RenewTokenHandler.java | 0 .../hadoop/ozone/shell/token/RenewerOption.java | 0 .../hadoop/ozone/shell/token/TokenCommands.java | 28 +- .../hadoop/ozone/shell/token/TokenHandler.java | 0 .../hadoop/ozone/shell/token/TokenOption.java | 0 .../hadoop/ozone/shell/token/package-info.java | 0 .../ozone/shell/volume/AddAclVolumeHandler.java | 0 .../ozone/shell/volume/ClearQuotaHandler.java | 0 .../ozone/shell/volume/CreateVolumeHandler.java | 0 .../ozone/shell/volume/DeleteVolumeHandler.java | 0 .../ozone/shell/volume/GetAclVolumeHandler.java | 0 .../ozone/shell/volume/InfoVolumeHandler.java | 0 .../ozone/shell/volume/ListVolumeHandler.java | 0 .../ozone/shell/volume/RemoveAclVolumeHandler.java | 0 .../ozone/shell/volume/SetAclVolumeHandler.java | 0 .../hadoop/ozone/shell/volume/SetQuotaHandler.java | 0 .../ozone/shell/volume/UpdateVolumeHandler.java | 0 .../hadoop/ozone/shell/volume/VolumeCommands.java | 27 +- .../hadoop/ozone/shell/volume/VolumeHandler.java | 0 .../hadoop/ozone/shell/volume/VolumeUri.java | 0 .../hadoop/ozone/shell/volume/package-info.java | 0 .../hadoop/ozone/shell/TestOzoneAddress.java | 0 .../shell/TestOzoneAddressClientCreation.java | 0 .../ozone/shell/keys/TestChecksumKeyHandler.java | 0 .../hadoop/ozone/shell/keys/package-info.java | 0 .../apache/hadoop/ozone/shell/package-info.java | 0 hadoop-ozone/client/pom.xml | 71 +- .../client/checksum/BaseFileChecksumHelper.java | 35 +- .../client/checksum/ECFileChecksumHelper.java | 29 - .../checksum/ReplicatedFileChecksumHelper.java | 46 - .../client/io/BlockDataStreamOutputEntry.java | 12 + .../client/io/BlockDataStreamOutputEntryPool.java | 29 + .../ozone/client/io/KeyDataStreamOutput.java | 20 +- .../ozone/client/io/OzoneDataStreamOutput.java | 70 +- .../hadoop/ozone/client/io/OzoneOutputStream.java | 6 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 52 +- hadoop-ozone/common/pom.xml | 148 ++- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 +- .../ozone/client/io/SelectorOutputStream.java | 17 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 11 - .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 11 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +- hadoop-ozone/csi/pom.xml | 199 ++-- .../org/apache/hadoop/ozone/csi/CsiServer.java | 2 +- hadoop-ozone/datanode/pom.xml | 38 +- hadoop-ozone/dev-support/checks/_build.sh | 16 +- hadoop-ozone/dev-support/checks/_lib.sh | 105 -- .../dev-support/checks/_mvn_unit_report.sh | 10 +- hadoop-ozone/dev-support/checks/acceptance.sh | 19 + hadoop-ozone/dev-support/checks/bats.sh | 6 +- hadoop-ozone/dev-support/checks/build.sh | 3 +- .../dev-support/checks/{build.sh => compile.sh} | 7 +- hadoop-ozone/dev-support/checks/docs.sh | 2 +- hadoop-ozone/dev-support/checks/findbugs.sh | 3 +- .../checks/{_build.sh => install/bats.sh} | 22 +- .../checks/{_build.sh => install/flekszible.sh} | 26 +- .../checks/{docs.sh => install/hugo.sh} | 56 +- .../checks/{native.sh => install/k3s.sh} | 15 +- .../checks/{native.sh => install/spotbugs.sh} | 13 +- hadoop-ozone/dev-support/checks/kubernetes.sh | 8 +- hadoop-ozone/dev-support/checks/native.sh | 2 +- hadoop-ozone/dev-support/checks/repro.sh | 17 +- .../dev-support/intellij/ozone-site-ha.xml | 4 - hadoop-ozone/dev-support/intellij/ozone-site.xml | 4 - .../intellij/runConfigurations/OzoneShell-ha.xml | 2 +- .../intellij/runConfigurations/OzoneShell.xml | 2 +- .../intellij/runConfigurations/ScmRoles.xml | 2 +- .../dist/dev-support/bin/dist-layout-stitching | 2 +- hadoop-ozone/dist/pom.xml | 226 ++-- .../Datanode Chunk Read_Write Dashboard.json | 85 +- .../dashboards/Ozone - OMComittedIndexMetrics.json | 166 +++ .../dist/src/main/compose/common/replicas-test.sh | 48 + .../src/main/compose/common/s3-haproxy-secure.yaml | 57 + .../dist/src/main/compose/common/s3-haproxy.cfg | 10 + .../dist/src/main/compose/common/s3a-test.sh | 2 +- .../src/main/compose/ozone-balancer/docker-config | 2 - .../dist/src/main/compose/ozone-ha/docker-config | 2 - .../src/main/compose/ozone-om-ha/docker-config | 1 - .../main/compose/ozone-om-prepare/docker-config | 1 - .../src/main/compose/ozone/docker-compose.yaml | 2 + .../dist/src/main/compose/ozone/prometheus.yml | 4 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 4 +- .../src/main/compose/ozonesecure-ha/docker-config | 2 - .../test.sh => ozonesecure/test-haproxy-s3g.sh} | 46 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 4 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 70 +- .../dist/src/main/compose/upgrade/README.md | 18 +- .../main/compose/upgrade/compose/ha/docker-config | 2 - .../src/main/compose/upgrade/compose/non-ha/.env | 22 - .../upgrade/compose/non-ha/docker-compose.yaml | 155 --- .../compose/upgrade/compose/non-ha/docker-config | 49 - .../main/compose/upgrade/compose/non-ha/load.sh | 29 - .../src/main/compose/upgrade/compose/om-ha/.env | 27 - .../upgrade/compose/om-ha/docker-compose.yaml | 178 --- .../compose/upgrade/compose/om-ha/docker-config | 65 -- .../src/main/compose/upgrade/compose/om-ha/load.sh | 30 - .../upgrade/upgrades/manual-upgrade/driver.sh | 54 - .../k8s/definitions/ozone/definitions/om-ha.yaml | 1 - .../k8s/definitions/ozone/definitions/scm-ha.yaml | 1 - .../k8s/examples/ozone-ha/config-configmap.yaml | 1 - .../dist/src/main/k8s/examples/test-all.sh | 23 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 5 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 1 + .../dist/src/main/smoketest/admincli/admin.robot | 2 +- .../src/main/smoketest/admincli/container.robot | 24 +- .../src/main/smoketest/admincli/datanode.robot | 2 +- .../lease-recovery.robot} | 10 +- .../src/main/smoketest/admincli/pipeline.robot | 2 +- .../smoketest/admincli/replicationmanager.robot | 2 +- .../src/main/smoketest/admincli/safemode.robot | 2 +- .../dist/src/main/smoketest/basic/links.robot | 47 +- .../src/main/smoketest/basic/ozone-shell-lib.robot | 39 +- .../smoketest/compatibility/dn-one-rocksdb.robot | 2 +- .../src/main/smoketest/compatibility/read.robot | 2 +- .../dist/src/main/smoketest/createmrenv.robot | 2 +- .../src/main/smoketest/debug/ozone-debug-ldb.robot | 8 +- .../src/main/smoketest/debug/ozone-debug.robot | 5 - .../dist/src/main/smoketest/ozone-lib/shell.robot | 5 + .../main/smoketest/recon/recon-taskstatus.robot | 125 +++ .../src/main/smoketest/s3/MultipartUpload.robot | 6 + .../dist/src/main/smoketest/s3/bucketcreate.robot | 12 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 14 + .../dist/src/main/smoketest/s3/objectputget.robot | 39 +- .../dist/src/main/smoketest/s3/rclone.robot | 46 + .../src/main/smoketest/s3/s3_compatbility_check.sh | 1 + .../src/main/smoketest/s3/secretgenerate.robot | 2 +- .../dist/src/main/smoketest/s3/secretrevoke.robot | 3 +- .../dist/src/main/smoketest/s3/webui.robot | 13 +- .../dist/src/main/smoketest/spnego/web.robot | 2 +- hadoop-ozone/dist/src/main/smoketest/testlib.sh | 50 + hadoop-ozone/dist/src/shell/ozone/ozone | 44 +- .../fault-injection-test/mini-chaos-tests/pom.xml | 33 +- .../fault-injection-test/network-tests/pom.xml | 25 +- hadoop-ozone/fault-injection-test/pom.xml | 11 +- hadoop-ozone/httpfsgateway/pom.xml | 178 ++- .../apache/ozone/fs/http/server/FSOperations.java | 2 + hadoop-ozone/insight/pom.xml | 77 +- .../ozone/insight/ConfigurationSubCommand.java | 2 +- .../apache/hadoop/ozone/insight/LogSubcommand.java | 2 +- .../hadoop/ozone/insight/MetricsSubCommand.java | 2 +- hadoop-ozone/integration-test/pom.xml | 229 ++-- .../AbstractContractGetFileStatusTest.java | 16 +- .../fs/ozone/AbstractOzoneFileSystemTest.java | 82 +- .../ozone/AbstractOzoneFileSystemTestWithFSO.java | 4 +- .../ozone/AbstractRootedOzoneFileSystemTest.java | 18 +- .../AbstractRootedOzoneFileSystemTestWithFSO.java | 4 +- .../ozone/TestDirectoryDeletingServiceWithFSO.java | 99 +- .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 36 +- .../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 2 - .../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 239 ++-- .../java/org/apache/hadoop/fs/ozone/TestO3FS.java | 2 +- .../apache/hadoop/fs/ozone/TestO3FSWithFSO.java | 2 +- .../hadoop/fs/ozone/TestO3FSWithFSPaths.java | 2 +- .../fs/ozone/TestO3FSWithFSPathsAndOMRatis.java | 28 - .../hadoop/fs/ozone/TestO3FSWithOMRatis.java | 28 - .../java/org/apache/hadoop/fs/ozone/TestOFS.java | 2 +- .../hadoop/fs/ozone/TestOFSWithCacheOnly.java | 28 - .../org/apache/hadoop/fs/ozone/TestOFSWithFSO.java | 2 +- .../fs/ozone/TestOFSWithFSOAndCacheOnly.java | 27 - .../hadoop/fs/ozone/TestOFSWithFSOAndOMRatis.java | 27 - .../TestOFSWithFSOAndOMRatisAndCacheOnly.java | 27 - .../apache/hadoop/fs/ozone/TestOFSWithFSPaths.java | 2 +- .../fs/ozone/TestOFSWithFSPathsAndOMRatis.java | 28 - .../ozone/TestOFSWithFSPathsAndOMRatisAndACL.java | 28 - .../TestOFSWithFSPathsAndOMRatisAndCacheOnly.java | 28 - .../apache/hadoop/fs/ozone/TestOFSWithOMRatis.java | 28 - .../fs/ozone/TestOzoneFileSystemPrefixParser.java | 2 +- .../fs/ozone/TestOzoneFileSystemWithStreaming.java | 8 +- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 51 +- .../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 2 - .../ozone/contract/AbstractOzoneContractTest.java | 62 +- .../fs/ozone/contract/TestOzoneContractFSO.java | 4 +- .../fs/ozone/contract/TestOzoneContractLegacy.java | 4 +- .../hadoop/hdds/scm/TestAllocateContainer.java | 26 +- .../hdds/scm/TestContainerReportWithKeys.java | 59 +- .../hadoop/hdds/scm/TestContainerSmallFile.java | 32 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 30 +- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 3 +- .../hadoop/hdds/scm/TestSCMNodeManagerMXBean.java | 36 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 1 - .../hadoop/hdds/scm/TestSecretKeySnapshot.java | 2 - .../apache/hadoop/hdds/scm/TestSecretKeysApi.java | 2 - .../hdds/scm/TestStorageContainerManager.java | 491 ++++----- .../hdds/scm/TestStorageContainerManagerHA.java | 2 - .../metrics/TestSCMContainerManagerMetrics.java | 1 + .../scm/pipeline/TestPipelineManagerMXBean.java | 25 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 7 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 28 - .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 46 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 3 - .../org/apache/hadoop/ozone/OzoneTestUtils.java | 33 + .../org/apache/hadoop/ozone/TestBlockTokens.java | 2 - .../apache/hadoop/ozone/TestBlockTokensCLI.java | 6 +- .../org/apache/hadoop/ozone/TestCpuMetrics.java | 18 +- .../apache/hadoop/ozone/TestDelegationToken.java | 6 +- .../ozone/TestGetClusterTreeInformation.java | 33 +- .../hadoop/ozone/TestMultipartObjectGet.java | 4 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 63 +- .../hadoop/ozone/UniformDatanodesFactory.java | 26 + .../om/lease}/TestLeaseRecoverer.java | 2 +- .../client/rpc/TestBlockDataStreamOutput.java | 401 ++++--- .../ozone/client/rpc/TestBlockOutputStream.java | 7 +- .../rpc/TestBlockOutputStreamWithFailures.java | 31 +- .../client/rpc/TestContainerStateMachine.java | 2 + .../ozone/client/rpc/TestDatanodeVersion.java | 143 --- .../client/rpc/TestDeleteWithInAdequateDN.java | 3 + .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 157 ++- .../ozone/client/rpc/TestOzoneRpcClient.java | 4 +- .../client/rpc/TestOzoneRpcClientWithRatis.java | 55 - .../client/rpc/read/TestChunkInputStream.java | 18 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 48 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 86 +- .../commandhandler/TestBlockDeletion.java | 50 +- .../commandhandler/TestDeleteContainerHandler.java | 6 + .../commandhandler/TestFinalizeBlock.java | 28 +- .../metrics/TestDatanodeQueueMetrics.java | 51 +- .../org/apache/hadoop/ozone/debug/TestLDBCli.java | 4 +- .../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 56 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 102 +- .../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 - .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 184 ++-- .../hadoop/ozone/om/TestOMEpochForNonRatis.java | 215 ---- .../apache/hadoop/ozone/om/TestObjectStore.java | 33 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 42 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 44 +- .../ozone/om/TestOzoneManagerConfiguration.java | 1 - .../ozone/om/TestOzoneManagerRestInterface.java | 24 +- .../hadoop/ozone/om/TestSecureOzoneManager.java | 6 +- .../om/ratis/TestOzoneManagerRatisRequest.java | 18 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 1 - ...TestSnapshotDeletingServiceIntegrationTest.java | 3 - .../TestReconInsightsForDeletedDirectories.java | 41 +- .../hadoop/ozone/recon/TestReconScmHASnapshot.java | 65 -- .../ozone/recon/TestReconScmNonHASnapshot.java | 64 -- .../apache/hadoop/ozone/recon/TestReconTasks.java | 1 + .../ozone/recon/TestReconWithOzoneManager.java | 4 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 2 - .../hadoop/ozone/repair/om/TestFSORepairTool.java | 281 +++-- .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 60 + .../hadoop/ozone/s3/awssdk/v1/TestS3SDKV1.java | 4 +- .../ozone/s3/awssdk/v1/TestS3SDKV1WithRatis.java | 49 - .../awssdk/v1/TestS3SDKV1WithRatisStreaming.java | 2 - .../ozone/shell/TestDeletedBlocksTxnShell.java | 2 - .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 2 +- .../hadoop/ozone/shell/TestOzoneDebugShell.java | 4 +- .../hadoop/ozone/shell/TestOzoneRepairShell.java | 139 ++- .../hadoop/ozone/shell/TestOzoneShellHA.java | 2 +- .../hadoop/ozone/shell/TestOzoneTenantShell.java | 48 +- .../hadoop/ozone/shell/TestReconfigShell.java | 2 +- .../apache/hadoop/ozone/shell/TestScmAdminHA.java | 41 +- .../ozone/shell/TestTransferLeadershipShell.java | 2 +- .../org/apache/ozone/test/ClusterForTests.java | 96 ++ .../test/java/org/apache/ozone/test/HATests.java | 83 ++ .../java/org/apache/ozone/test/NonHATests.java | 141 +++ .../test/TestOzoneIntegrationHA.java} | 18 +- .../test/TestOzoneIntegrationNonHA.java} | 16 +- .../java/org/apache/ozone/test}/package-info.java | 10 +- hadoop-ozone/interface-client/pom.xml | 92 +- hadoop-ozone/interface-storage/pom.xml | 59 +- hadoop-ozone/ozone-manager/pom.xml | 387 +++---- .../apache/hadoop/ozone/om/DeleteKeysResult.java | 36 +- .../hadoop/ozone/om/DeletingServiceMetrics.java | 163 +++ .../org/apache/hadoop/ozone/om/KeyManager.java | 10 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 65 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 46 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 1 - .../hadoop/ozone/om/OmMetadataManagerImpl.java | 7 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +- .../hadoop/ozone/om/OzoneManagerStarter.java | 5 +- .../execution/flowcontrol/ExecutionContext.java} | 42 +- .../om/execution/flowcontrol}/package-info.java | 4 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 4 +- .../hadoop/ozone/om/request/OMClientRequest.java | 8 +- .../om/request/bucket/OMBucketCreateRequest.java | 6 +- .../om/request/bucket/OMBucketDeleteRequest.java | 6 +- .../om/request/bucket/OMBucketSetOwnerRequest.java | 6 +- .../request/bucket/OMBucketSetPropertyRequest.java | 6 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 6 +- .../request/bucket/acl/OMBucketAddAclRequest.java | 6 +- .../bucket/acl/OMBucketRemoveAclRequest.java | 6 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 6 +- .../om/request/file/OMDirectoryCreateRequest.java | 6 +- .../file/OMDirectoryCreateRequestWithFSO.java | 6 +- .../ozone/om/request/file/OMFileCreateRequest.java | 6 +- .../request/file/OMFileCreateRequestWithFSO.java | 6 +- .../om/request/file/OMRecoverLeaseRequest.java | 6 +- .../om/request/key/OMAllocateBlockRequest.java | 6 +- .../request/key/OMAllocateBlockRequestWithFSO.java | 6 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 25 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 6 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 6 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 6 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeyPurgeRequest.java | 16 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 6 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeySetTimesRequest.java | 6 +- .../request/key/OMKeySetTimesRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 6 +- .../ozone/om/request/key/OMKeysRenameRequest.java | 6 +- .../om/request/key/OMOpenKeysDeleteRequest.java | 6 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 6 +- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 6 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 6 +- .../request/key/acl/OMKeyAddAclRequestWithFSO.java | 6 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 6 +- .../key/acl/OMKeyRemoveAclRequestWithFSO.java | 6 +- .../om/request/key/acl/OMKeySetAclRequest.java | 6 +- .../request/key/acl/OMKeySetAclRequestWithFSO.java | 6 +- .../request/key/acl/prefix/OMPrefixAclRequest.java | 6 +- .../S3ExpiredMultipartUploadsAbortRequest.java | 6 +- .../S3InitiateMultipartUploadRequest.java | 6 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 6 +- .../multipart/S3MultipartUploadAbortRequest.java | 6 +- .../S3MultipartUploadCommitPartRequest.java | 6 +- .../S3MultipartUploadCompleteRequest.java | 6 +- .../om/request/s3/security/OMSetSecretRequest.java | 6 +- .../om/request/s3/security/S3GetSecretRequest.java | 14 +- .../request/s3/security/S3RevokeSecretRequest.java | 4 +- .../s3/tagging/S3DeleteObjectTaggingRequest.java | 6 +- .../S3DeleteObjectTaggingRequestWithFSO.java | 6 +- .../s3/tagging/S3PutObjectTaggingRequest.java | 6 +- .../tagging/S3PutObjectTaggingRequestWithFSO.java | 6 +- .../tenant/OMSetRangerServiceVersionRequest.java | 6 +- .../s3/tenant/OMTenantAssignAdminRequest.java | 6 +- .../tenant/OMTenantAssignUserAccessIdRequest.java | 6 +- .../request/s3/tenant/OMTenantCreateRequest.java | 6 +- .../request/s3/tenant/OMTenantDeleteRequest.java | 6 +- .../s3/tenant/OMTenantRevokeAdminRequest.java | 6 +- .../tenant/OMTenantRevokeUserAccessIdRequest.java | 6 +- .../security/OMCancelDelegationTokenRequest.java | 6 +- .../security/OMGetDelegationTokenRequest.java | 6 +- .../security/OMRenewDelegationTokenRequest.java | 6 +- .../request/snapshot/OMSnapshotCreateRequest.java | 8 +- .../request/snapshot/OMSnapshotDeleteRequest.java | 6 +- .../snapshot/OMSnapshotMoveDeletedKeysRequest.java | 12 +- .../snapshot/OMSnapshotMoveTableKeysRequest.java | 12 +- .../request/snapshot/OMSnapshotPurgeRequest.java | 14 +- .../request/snapshot/OMSnapshotRenameRequest.java | 9 +- .../snapshot/OMSnapshotSetPropertyRequest.java | 6 +- .../om/request/upgrade/OMCancelPrepareRequest.java | 8 +- .../request/upgrade/OMFinalizeUpgradeRequest.java | 6 +- .../ozone/om/request/upgrade/OMPrepareRequest.java | 10 +- .../om/request/util/OMEchoRPCWriteRequest.java | 4 +- .../om/request/validation/VersionExtractor.java | 63 ++ .../om/request/volume/OMQuotaRepairRequest.java | 6 +- .../om/request/volume/OMVolumeCreateRequest.java | 6 +- .../om/request/volume/OMVolumeDeleteRequest.java | 6 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 6 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 6 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 6 +- .../volume/acl/OMVolumeRemoveAclRequest.java | 6 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 6 +- .../S3MultipartUploadCompleteResponse.java | 2 + .../S3MultipartUploadCompleteResponseWithFSO.java | 13 +- .../response/s3/security/S3GetSecretResponse.java | 9 +- .../response/snapshot/OMSnapshotPurgeResponse.java | 6 + .../om/service/AbstractKeyDeletingService.java | 67 +- .../ozone/om/service/DirectoryDeletingService.java | 52 +- .../ozone/om/service/KeyDeletingService.java | 5 + ...OzoneManagerProtocolServerSideTranslatorPB.java | 19 +- .../protocolPB/OzoneManagerRequestHandler.java | 12 +- .../hadoop/ozone/protocolPB/RequestHandler.java | 14 +- .../OzoneDelegationTokenSecretManager.java | 5 +- .../webapps/ozoneManager/om-overview.html | 3 + .../hadoop/ozone/om/TestOmSnapshotManager.java | 60 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 10 +- .../ozone/om/request/key/TestOMKeyRequest.java | 4 + .../s3/security/TestS3GetSecretRequest.java | 28 + .../TestOMSnapshotPurgeRequestAndResponse.java | 37 + .../request/validation/TestVersionExtractor.java | 66 ++ .../om/service/TestDirectoryDeletingService.java | 62 +- .../security/TestOmCertificateClientInit.java | 10 +- hadoop-ozone/ozonefs-common/pom.xml | 100 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 8 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 7 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 8 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 7 +- .../fs/ozone/CapableOzoneFSDataStreamOutput.java | 63 ++ .../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 16 + .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 5 + hadoop-ozone/ozonefs-hadoop2/pom.xml | 68 +- hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 45 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 21 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 6 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 + hadoop-ozone/ozonefs-shaded/pom.xml | 94 +- hadoop-ozone/ozonefs/pom.xml | 117 +- .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 6 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 6 + hadoop-ozone/pom.xml | 244 +++-- hadoop-ozone/recon-codegen/pom.xml | 48 +- .../recon/schema/ReconTaskSchemaDefinition.java | 2 + hadoop-ozone/recon/pom.xml | 693 ++++++------ .../hadoop/ozone/recon/ReconControllerModule.java | 2 + .../org/apache/hadoop/ozone/recon/ReconServer.java | 5 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 6 +- .../hadoop/ozone/recon/api/TaskStatusService.java | 5 +- .../ozone/recon/fsck/ContainerHealthTask.java | 23 +- .../hadoop/ozone/recon/scm/PipelineSyncTask.java | 22 +- .../ozone/recon/scm/ReconDeadNodeHandler.java | 4 +- .../hadoop/ozone/recon/scm/ReconScmTask.java | 44 +- .../ozone/recon/scm/ReconStaleNodeHandler.java | 2 +- .../scm/ReconStorageContainerManagerFacade.java | 34 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 127 ++- .../ozone/recon/tasks/ContainerKeyMapperTask.java | 6 +- .../ozone/recon/tasks/ContainerSizeCountTask.java | 61 +- .../ozone/recon/tasks/FileSizeCountTask.java | 4 +- .../hadoop/ozone/recon/tasks/NSSummaryTask.java | 3 + .../ozone/recon/tasks/OMDBUpdatesHandler.java | 9 + .../ozone/recon/tasks/OMUpdateEventBatch.java | 10 +- .../ozone/recon/tasks/OmTableInsightTask.java | 4 +- .../ozone/recon/tasks/ReconTaskController.java | 14 +- .../ozone/recon/tasks/ReconTaskControllerImpl.java | 263 +++-- .../ozone/recon/tasks/types/NamedCallableTask.java | 34 +- .../recon/tasks/types/TaskExecutionException.java | 18 +- .../ozone/recon/tasks/types}/package-info.java | 7 +- .../tasks/updater/ReconTaskStatusUpdater.java | 123 +++ .../updater/ReconTaskStatusUpdaterManager.java | 69 ++ .../ozone/recon/tasks/updater}/package-info.java | 7 +- .../ozone/recon/upgrade/ReconLayoutFeature.java | 3 +- .../upgrade/ReconTaskStatusTableUpgradeAction.java | 104 ++ .../webapps/recon/ozone-recon-web/package.json | 1 + .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 12 + .../src/__tests__/datanodes/Datanodes.test.tsx | 191 ++++ .../__tests__/datanodes/DatanodesTable.test.tsx | 151 +++ .../src/__tests__/locators/locators.ts | 13 +- .../mocks/datanodeMocks/datanodeResponseMocks.ts | 212 ++++ .../mocks/datanodeMocks/datanodeServer.ts | 72 ++ .../src/__tests__/utils/datanodes.utils.tsx} | 10 +- .../webapps/recon/ozone-recon-web/src/app.tsx | 9 +- .../src/components/overviewCard/overviewCard.tsx | 9 +- .../recon/ozone-recon-web/src/utils/common.tsx | 29 + .../components/duBreadcrumbNav/duBreadcrumbNav.tsx | 8 +- .../src/v2/components/duMetadata/duMetadata.tsx | 340 +++--- .../overviewCard/overviewStorageCard.tsx | 8 +- .../src/v2/components/plots/duPieChart.tsx | 2 +- .../src/v2/components/search/search.tsx | 6 +- .../src/v2/components/select/multiSelect.tsx | 6 +- .../src/v2/components/storageBar/storageBar.less | 29 +- .../src/v2/components/storageBar/storageBar.tsx | 46 +- .../src/v2/components/tables/datanodesTable.tsx | 8 +- .../src/v2/pages/datanodes/datanodes.tsx | 13 +- .../src/v2/pages/diskUsage/diskUsage.less | 1 + .../src/v2/pages/diskUsage/diskUsage.tsx | 25 +- .../src/v2/pages/overview/overview.tsx | 27 +- .../src/views/overview/overview.tsx | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 2 +- .../ozone/recon/api/TestTaskStatusService.java | 21 +- .../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 16 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 34 +- .../TestReconInternalSchemaDefinition.java | 6 +- .../persistence/TestReconWithDifferentSqlDBs.java | 2 +- .../recon/persistence/TestSqlSchemaSetup.java | 10 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 115 +- .../recon/tasks/TestContainerKeyMapperTask.java | 6 +- .../recon/tasks/TestContainerSizeCountTask.java | 22 +- .../ozone/recon/tasks/TestFileSizeCountTask.java | 8 +- .../ozone/recon/tasks/TestNSSummaryTask.java | 2 +- .../recon/tasks/TestNSSummaryTaskWithFSO.java | 2 +- .../recon/tasks/TestNSSummaryTaskWithLegacy.java | 2 +- .../TestNSSummaryTaskWithLegacyOBSLayout.java | 2 +- .../recon/tasks/TestNSSummaryTaskWithOBS.java | 2 +- .../ozone/recon/tasks/TestOmTableInsightTask.java | 18 +- .../recon/tasks/TestReconTaskControllerImpl.java | 53 +- .../recon/tasks/TestReconTaskStatusUpdater.java | 101 ++ hadoop-ozone/s3-secret-store/pom.xml | 30 +- .../ozone/s3/remote/vault/VaultS3SecretStore.java | 8 +- hadoop-ozone/s3gateway/pom.xml | 239 ++-- .../hadoop/ozone/s3/AuthorizationFilter.java | 6 - .../java/org/apache/hadoop/ozone/s3/Gateway.java | 11 +- .../apache/hadoop/ozone/s3/GatewayApplication.java | 2 +- .../hadoop/ozone/s3/RootPageDisplayFilter.java | 64 -- .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 14 + .../hadoop/ozone/s3/S3GatewayHttpServer.java | 80 +- ...ttpServer.java => S3GatewayWebAdminServer.java} | 59 +- .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 7 - .../hadoop/ozone/s3/endpoint/EndpointBase.java | 12 + .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 5 +- .../Application.java} | 8 +- .../ozone/s3secret/S3SecretManagementEndpoint.java | 2 +- .../webapps/{s3gateway => s3g-web}/WEB-INF/web.xml | 30 +- .../webapps/{static => s3g-web}/images/ozone.ico | Bin .../webapps/{static => s3g-web}/index.html | 24 +- .../resources/webapps/s3gateway/WEB-INF/web.xml | 11 +- .../hadoop/ozone/client/OzoneBucketStub.java | 10 + .../ozone/client/OzoneDataStreamOutputStub.java | 2 +- .../hadoop/ozone/s3/TestAuthorizationFilter.java | 22 - .../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 39 +- .../ozone/s3/TestVirtualHostStyleFilter.java | 23 - .../hadoop/ozone/s3/endpoint/EndpointBuilder.java | 125 +++ .../ozone/s3/endpoint/ObjectEndpointBuilder.java} | 25 +- .../s3/endpoint/TestAbortMultipartUpload.java | 10 +- .../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 5 +- .../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 5 +- .../hadoop/ozone/s3/endpoint/TestBucketHead.java | 5 +- .../hadoop/ozone/s3/endpoint/TestBucketList.java | 26 +- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 5 +- .../s3/endpoint/TestInitiateMultipartUpload.java | 10 +- .../hadoop/ozone/s3/endpoint/TestListParts.java | 34 +- .../s3/endpoint/TestMultipartUploadComplete.java | 35 +- .../s3/endpoint/TestMultipartUploadWithCopy.java | 2 + .../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 7 +- .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 9 +- .../hadoop/ozone/s3/endpoint/TestObjectHead.java | 6 +- .../ozone/s3/endpoint/TestObjectMultiDelete.java | 10 +- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 13 +- .../ozone/s3/endpoint/TestObjectTaggingDelete.java | 16 +- .../ozone/s3/endpoint/TestObjectTaggingGet.java | 11 +- .../ozone/s3/endpoint/TestObjectTaggingPut.java | 19 +- .../hadoop/ozone/s3/endpoint/TestPartUpload.java | 51 +- .../s3/endpoint/TestPartUploadWithStream.java | 34 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 90 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 5 +- .../ozone/s3/endpoint/TestUploadWithStream.java | 42 +- .../ozone/s3/metrics/TestS3GatewayMetrics.java | 18 +- hadoop-ozone/tools/pom.xml | 211 ++-- .../org/apache/hadoop/ozone/admin}/OzoneAdmin.java | 17 +- .../ozone/admin/nssummary/NSSummaryAdmin.java | 6 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 6 +- .../{debug => admin/om/lease}/LeaseRecoverer.java | 13 +- .../om/lease/LeaseSubCommand.java} | 17 +- .../hadoop/ozone/admin/om/lease/package-info.java} | 9 +- .../ozone/admin/reconfig/ReconfigureCommands.java | 17 +- .../ozone/admin/scm/DeletedBlocksTxnCommands.java | 13 +- .../apache/hadoop/ozone/admin/scm/ScmAdmin.java | 2 +- .../org/apache/hadoop/ozone/conf/OzoneGetConf.java | 26 +- .../org/apache/hadoop/ozone/debug/OzoneDebug.java | 5 +- .../apache/hadoop/ozone/debug/ReadReplicas.java | 9 +- .../datanode/DatanodeDebug.java} | 29 +- .../container/ContainerCommands.java | 32 +- .../{ => datanode}/container/ExportSubcommand.java | 4 +- .../{ => datanode}/container/InfoSubcommand.java | 6 +- .../container/InspectSubcommand.java | 9 +- .../{ => datanode}/container/ListSubcommand.java | 6 +- .../{ => datanode}/container/package-info.java | 2 +- .../debug/{chunk => datanode}/package-info.java | 4 +- .../apache/hadoop/ozone/debug/ldb/Checkpoint.java | 63 ++ .../apache/hadoop/ozone/debug/ldb/DBScanner.java | 28 +- .../apache/hadoop/ozone/debug/ldb/RDBParser.java | 18 +- .../apache/hadoop/ozone/debug/ldb/ValueSchema.java | 26 +- .../om/OMDebug.java} | 28 +- .../hadoop/ozone/debug/{ => om}/PrefixParser.java | 12 +- .../{repair/quota => debug/om}/package-info.java | 4 +- .../replicas/ReplicasDebug.java} | 34 +- .../{ => replicas}/chunk/ChunkDataNodeDetails.java | 2 +- .../debug/{ => replicas}/chunk/ChunkDetails.java | 2 +- .../{ => replicas}/chunk/ChunkKeyHandler.java | 20 +- .../debug/{ => replicas}/chunk/ChunkType.java | 2 +- .../{ => replicas}/chunk/ContainerChunkInfo.java | 2 +- .../debug/{ => replicas}/chunk/package-info.java | 4 +- .../debug/{chunk => replicas}/package-info.java | 4 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 5 +- .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 2 + .../ozone/freon/ClosedContainerReplicator.java | 40 +- .../hadoop/ozone/freon/DNRPCLoadGenerator.java | 6 +- .../hadoop/ozone/freon/DatanodeBlockPutter.java | 2 + .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 2 + .../hadoop/ozone/freon/DatanodeChunkValidator.java | 2 + .../hadoop/ozone/freon/DatanodeSimulator.java | 6 +- .../freon/FollowerAppendLogEntryGenerator.java | 2 + .../java/org/apache/hadoop/ozone/freon/Freon.java | 51 +- .../FreonSubcommand.java} | 9 +- .../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 2 + .../hadoop/ozone/freon/HadoopFsGenerator.java | 2 + .../hadoop/ozone/freon/HadoopFsValidator.java | 2 + .../ozone/freon/HadoopNestedDirGenerator.java | 2 + .../apache/hadoop/ozone/freon/HsyncGenerator.java | 4 +- .../ozone/freon/LeaderAppendLogEntryGenerator.java | 2 + .../hadoop/ozone/freon/OmBucketGenerator.java | 2 + .../ozone/freon/OmBucketReadWriteFileOps.java | 3 +- .../ozone/freon/OmBucketReadWriteKeyOps.java | 3 +- .../apache/hadoop/ozone/freon/OmBucketRemover.java | 2 + .../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 + .../hadoop/ozone/freon/OmMetadataGenerator.java | 2 + .../hadoop/ozone/freon/OmRPCLoadGenerator.java | 2 + .../hadoop/ozone/freon/OzoneClientCreator.java | 2 + .../ozone/freon/OzoneClientKeyGenerator.java | 2 + .../freon/OzoneClientKeyReadWriteListOps.java | 2 + .../hadoop/ozone/freon/OzoneClientKeyRemover.java | 2 + .../ozone/freon/OzoneClientKeyValidator.java | 2 + .../hadoop/ozone/freon/RandomKeyGenerator.java | 8 +- .../hadoop/ozone/freon/RangeKeysGenerator.java | 2 + .../hadoop/ozone/freon/S3BucketGenerator.java | 2 + .../apache/hadoop/ozone/freon/S3KeyGenerator.java | 2 + .../hadoop/ozone/freon/SCMThroughputBenchmark.java | 9 +- .../apache/hadoop/ozone/freon/SameKeyReader.java | 2 + .../hadoop/ozone/freon/StreamingGenerator.java | 2 + .../containergenerator/GeneratorDatanode.java | 3 + .../freon/containergenerator/GeneratorOm.java | 4 + .../freon/containergenerator/GeneratorScm.java | 4 + .../GenerateOzoneRequiredConfigurations.java | 48 +- .../apache/hadoop/ozone/repair/OzoneRepair.java | 31 +- .../ReadOnlyCommand.java} | 9 +- .../org/apache/hadoop/ozone/repair/RepairTool.java | 150 +++ .../repair/{ldb => }/TransactionInfoRepair.java | 94 +- .../apache/hadoop/ozone/repair/ldb/RDBRepair.java | 57 - .../hadoop/ozone/repair/om/FSORepairCLI.java | 78 -- .../hadoop/ozone/repair/om/FSORepairTool.java | 730 +++++++------ .../apache/hadoop/ozone/repair/om/OMRepair.java | 20 +- .../SnapshotChainRepair.java} | 91 +- .../om/SnapshotRepair.java} | 16 +- .../ozone/repair/{ => om}/quota/QuotaRepair.java | 32 +- .../ozone/repair/{ => om}/quota/QuotaStatus.java | 17 +- .../ozone/repair/{ => om}/quota/QuotaTrigger.java | 34 +- .../repair/{ldb => om/quota}/package-info.java | 4 +- .../{om/OMRepair.java => scm/SCMRepair.java} | 29 +- .../scm/cert/CertRepair.java} | 17 +- .../{ => scm/cert}/RecoverSCMCertificate.java | 70 +- .../repair/{ldb => scm/cert}/package-info.java | 4 +- .../ozone/repair/{ldb => scm}/package-info.java | 4 +- .../org/apache/hadoop/ozone/shell/OzoneRatis.java | 2 +- .../ozone/shell/checknative/CheckNative.java | 11 +- .../hadoop/ozone/conf/TestGetConfOptions.java | 76 +- .../hadoop/ozone/repair/TestOzoneRepair.java | 65 +- .../ozone/repair/TestTransactionInfoRepair.java | 158 +++ .../repair/ldb/TestTransactionInfoRepair.java | 146 --- .../ozone/repair/om/TestSnapshotChainRepair.java | 304 ++++++ .../ozone/scm/TestDecommissionScmSubcommand.java | 2 +- pom.xml | 176 ++- 921 files changed, 15572 insertions(+), 14057 deletions(-) diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java index 16d282da5a,11ef3e9c18..38ffbf5d50 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeConfiguration.java @@@ -581,40 -573,20 +581,54 @@@ public class DatanodeConfiguration exte private boolean bCheckEmptyContainerDir = OZONE_DATANODE_CHECK_EMPTY_CONTAINER_DIR_ON_DELETE_DEFAULT; + /** + * Whether to check container directory or not to determine + * container is empty. + */ + @Config(key = "container.checksum.lock.stripes", + type = ConfigType.INT, + defaultValue = "127", + tags = { DATANODE }, + description = "The number of lock stripes used to coordinate modifications to container checksum information. " + + "This information is only updated after a container is closed and does not affect the data read or write" + + " path. Each container in the datanode will be mapped to one lock which will only be held while its " + + "checksum information is updated." + ) + private int containerChecksumLockStripes = CONTAINER_CHECKSUM_LOCK_STRIPES_DEFAULT; + + @Config(key = "container.client.cache.size", + type = ConfigType.INT, + defaultValue = "100", + tags = { DATANODE }, + description = "The maximum number of clients to be cached by the datanode client manager" + ) + private int containerClientCacheSize = CONTAINER_CLIENT_CACHE_SIZE_DEFAULT; + + @Config(key = "container.client.cache.stale.threshold", + type = ConfigType.INT, + defaultValue = "10000", + tags = { DATANODE }, + description = "The stale threshold in ms for a client in cache. After this threshold the client " + + "is evicted from cache." + ) + private int containerClientCacheStaleThreshold = + CONTAINER_CLIENT_CACHE_STALE_THRESHOLD_MILLISECONDS_DEFAULT; + + @Config(key = "delete.container.timeout", + type = ConfigType.TIME, + defaultValue = "60s", + tags = { DATANODE }, + description = "If a delete container request spends more than this time waiting on the container lock or " + + "performing pre checks, the command will be skipped and SCM will resend it automatically. This avoids " + + "commands running for a very long time without SCM being informed of the progress." + ) + private long deleteContainerTimeoutMs = Duration.ofSeconds(60).toMillis(); + + public long getDeleteContainerTimeoutMs() { + return deleteContainerTimeoutMs; + } + + @SuppressWarnings("checkstyle:MethodLength") @PostConstruct public void validate() { if (containerDeleteThreads < 1) { diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java index 06987f6356,267a2ecb66..3e3df12d40 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueHandler.java @@@ -41,8 -41,8 +42,9 @@@ import org.apache.hadoop.fs.FileUtil import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.conf.ConfigurationSource; + import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandRequestProto; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerCommandResponseProto; @@@ -60,14 -60,10 +62,15 @@@ import org.apache.hadoop.hdds.utils.Fau import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.common.Checksum; +import org.apache.hadoop.ozone.common.ChecksumByteBuffer; +import org.apache.hadoop.ozone.common.ChecksumByteBufferFactory; import org.apache.hadoop.ozone.common.ChunkBuffer; + import org.apache.hadoop.ozone.common.ChunkBufferToByteString; import org.apache.hadoop.ozone.common.OzoneChecksumException; import org.apache.hadoop.ozone.common.utils.BufferUtils; +import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager; +import org.apache.hadoop.ozone.container.checksum.DNContainerOperationClient; +import org.apache.hadoop.ozone.container.checksum.ContainerMerkleTree; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; @@@ -135,10 -125,9 +138,11 @@@ import static org.apache.hadoop.hdds.sc import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.putBlockResponseSuccess; import static org.apache.hadoop.hdds.scm.protocolPB.ContainerCommandResponseBuilders.unsupportedRequest; import static org.apache.hadoop.hdds.scm.utils.ClientCommandsUtils.getReadChunkVersion; + + +import org.apache.hadoop.ozone.container.common.interfaces.ScanResult; import static org.apache.hadoop.ozone.OzoneConsts.INCREMENTAL_CHUNK_LIST; + import static org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.DEFAULT_LAYOUT; -import static org.apache.hadoop.ozone.container.common.interfaces.Container.ScanResult; import org.apache.hadoop.util.Time; import org.apache.ratis.statemachine.StateMachine; @@@ -162,17 -152,27 +167,31 @@@ public class KeyValueHandler extends Ha private final boolean validateChunkChecksumData; // A striped lock that is held during container creation. private final Striped<Lock> containerCreationLocks; + private final ContainerChecksumTreeManager checksumManager; private static FaultInjector injector; + private final Clock clock; public KeyValueHandler(ConfigurationSource config, String datanodeId, ContainerSet contSet, VolumeSet volSet, ContainerMetrics metrics, - IncrementalReportSender<Container> icrSender) { - this(config, datanodeId, contSet, volSet, metrics, icrSender, Clock.systemUTC()); + IncrementalReportSender<Container> icrSender, + ContainerChecksumTreeManager checksumManager) { ++ this(config, datanodeId, contSet, volSet, metrics, icrSender, checksumManager, Clock.systemUTC()); + } + ++ @SuppressWarnings("checkstyle:ParameterNumber") + public KeyValueHandler(ConfigurationSource config, + String datanodeId, + ContainerSet contSet, + VolumeSet volSet, + ContainerMetrics metrics, + IncrementalReportSender<Container> icrSender, ++ ContainerChecksumTreeManager checksumManager, + Clock clock) { super(config, datanodeId, contSet, volSet, metrics, icrSender); + this.clock = clock; blockManager = new BlockManagerImpl(config); validateChunkChecksumData = conf.getObject( DatanodeConfiguration.class).isChunkDataValidationCheck(); diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index 993179d1b7,073cbfa6ed..b630fa11e7 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@@ -48,8 -49,8 +49,9 @@@ import org.apache.hadoop.ozone.OzoneCon import org.apache.hadoop.ozone.common.Checksum; import org.apache.hadoop.ozone.common.ChecksumData; import org.apache.hadoop.ozone.common.ChunkBuffer; + import org.apache.hadoop.ozone.common.ChunkBufferToByteString; import org.apache.hadoop.ozone.container.ContainerTestHelper; +import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index 0ff2aaa22b,83a6cddf4d..80d74532f2 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@@ -22,7 -22,7 +22,8 @@@ import java.io.File import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.EnumSet; + import java.time.Clock; import java.util.List; import java.util.Collections; import java.util.HashMap; @@@ -53,7 -47,7 +54,8 @@@ import org.apache.hadoop.ozone.containe import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.Handler; +import org.apache.hadoop.ozone.container.common.report.IncrementalReportSender; + import org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration; import org.apache.hadoop.ozone.container.common.statemachine.StateContext; import org.apache.hadoop.ozone.container.common.utils.StorageVolumeUtil; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; @@@ -64,12 -58,8 +66,13 @@@ import org.apache.ozone.test.GenericTes import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_VOLUME_CHOOSING_POLICY; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; +import static org.apache.hadoop.hdds.protocol.MockDatanodeDetails.randomDatanodeDetails; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.QUASI_CLOSED; +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.UNHEALTHY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; +import static org.apache.hadoop.ozone.OzoneConsts.GB; + import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@@ -303,6 -295,13 +306,13 @@@ public class TestKeyValueHandler keyValueHandler.getVolumeChoosingPolicyForTesting() .getClass().getName()); + // Ensures that KeyValueHandler falls back to FILE_PER_BLOCK. + conf.set(OZONE_SCM_CONTAINER_LAYOUT_KEY, "FILE_PER_CHUNK"); - new KeyValueHandler(conf, context.getParent().getDatanodeDetails().getUuidString(), cset, volumeSet, - metrics, c -> { }); ++ ContainerTestUtils.getKeyValueHandler(conf, ++ context.getParent().getDatanodeDetails().getUuidString(), cset, volumeSet); + assertEquals(ContainerLayoutVersion.FILE_PER_BLOCK, + conf.getEnum(OZONE_SCM_CONTAINER_LAYOUT_KEY, ContainerLayoutVersion.FILE_PER_CHUNK)); + //Set a class which is not of sub class of VolumeChoosingPolicy conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY, "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher"); @@@ -464,68 -458,70 +474,131 @@@ } } + @ContainerLayoutTestInfo.ContainerTest + public void testReconcileContainer(ContainerLayoutVersion layoutVersion) throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + + KeyValueContainerData data = new KeyValueContainerData(123L, layoutVersion, GB, + PipelineID.randomId().toString(), randomDatanodeDetails().getUuidString()); + + Container container = new KeyValueContainer(data, conf); + ContainerSet containerSet = new ContainerSet(1000); + containerSet.addContainer(container); + + // Allows checking the invocation count of the lambda. + AtomicInteger icrCount = new AtomicInteger(0); + IncrementalReportSender<Container> icrSender = c -> { + // Check that the ICR contains expected info about the container. + ContainerReplicaProto report = c.getContainerReport(); + long reportedID = report.getContainerID(); + Assertions.assertEquals(container.getContainerData().getContainerID(), reportedID); + + long reportDataChecksum = report.getDataChecksum(); + Assertions.assertNotEquals(0, reportDataChecksum, + "Container report should have populated the checksum field with a non-zero value."); + icrCount.incrementAndGet(); + }; + + KeyValueHandler keyValueHandler = new KeyValueHandler(conf, randomDatanodeDetails().getUuidString(), containerSet, + mock(MutableVolumeSet.class), mock(ContainerMetrics.class), icrSender, new ContainerChecksumTreeManager(conf)); + + Assertions.assertEquals(0, icrCount.get()); + // This should trigger container report validation in the ICR handler above. + keyValueHandler.reconcileContainer(mock(DNContainerOperationClient.class), container, Collections.emptySet()); + Assertions.assertEquals(1, icrCount.get()); + } + + @Test + public void testGetContainerChecksumInfoOnInvalidContainerStates() { + when(handler.handleGetContainerChecksumInfo(any(), any())).thenCallRealMethod(); + + // Only mock what is necessary for the request to fail. This test does not cover allowed states. + KeyValueContainer container = mock(KeyValueContainer.class); + KeyValueContainerData containerData = mock(KeyValueContainerData.class); + when(container.getContainerData()).thenReturn(containerData); + + ContainerCommandRequestProto request = mock(ContainerCommandRequestProto.class); + when(request.hasGetContainerChecksumInfo()).thenReturn(true); + when(request.getCmdType()).thenReturn(ContainerProtos.Type.GetContainerChecksumInfo); + when(request.getTraceID()).thenReturn("123"); + + Set<State> disallowedStates = EnumSet.allOf(State.class); + disallowedStates.removeAll(EnumSet.of(CLOSED, QUASI_CLOSED, UNHEALTHY)); + - for (State state: disallowedStates) { ++ for (State state : disallowedStates) { + when(containerData.getState()).thenReturn(state); + ContainerProtos.ContainerCommandResponseProto response = handler.handleGetContainerChecksumInfo(request, + container); + assertNotNull(response); + assertEquals(ContainerProtos.Result.UNCLOSED_CONTAINER_IO, response.getResult()); + assertTrue(response.getMessage().contains(state.toString()), "Response message did not contain the container " + + "state " + state); + } + } + @Test + public void testDeleteContainerTimeout() throws IOException { + final String testDir = tempDir.toString(); + final long containerID = 1L; + final String clusterId = UUID.randomUUID().toString(); + final String datanodeId = UUID.randomUUID().toString(); + final ConfigurationSource conf = new OzoneConfiguration(); + final ContainerSet containerSet = new ContainerSet(1000); + final MutableVolumeSet volumeSet = mock(MutableVolumeSet.class); + final Clock clock = mock(Clock.class); + long startTime = System.currentTimeMillis(); + + DatanodeConfiguration dnConf = conf.getObject(DatanodeConfiguration.class); + when(clock.millis()) + .thenReturn(startTime) + .thenReturn(startTime + dnConf.getDeleteContainerTimeoutMs() + 1); + + HddsVolume hddsVolume = new HddsVolume.Builder(testDir).conf(conf) + .clusterID(clusterId).datanodeUuid(datanodeId) + .volumeSet(volumeSet) + .build(); + hddsVolume.format(clusterId); + hddsVolume.createWorkingDir(clusterId, null); + hddsVolume.createTmpDirs(clusterId); + + when(volumeSet.getVolumesList()) + .thenReturn(Collections.singletonList(hddsVolume)); + + List<HddsVolume> hddsVolumeList = StorageVolumeUtil + .getHddsVolumesList(volumeSet.getVolumesList()); + + assertEquals(1, hddsVolumeList.size()); + + final ContainerMetrics metrics = ContainerMetrics.create(conf); + + final AtomicInteger icrReceived = new AtomicInteger(0); + + final KeyValueHandler kvHandler = new KeyValueHandler(conf, + datanodeId, containerSet, volumeSet, metrics, - c -> icrReceived.incrementAndGet(), clock); ++ c -> icrReceived.incrementAndGet(), new ContainerChecksumTreeManager(conf), clock); + kvHandler.setClusterID(clusterId); + + final ContainerCommandRequestProto createContainer = + createContainerRequest(datanodeId, containerID); + kvHandler.handleCreateContainer(createContainer, null); + assertEquals(1, icrReceived.get()); + assertNotNull(containerSet.getContainer(containerID)); + + // The delete should not have gone through due to the mocked clock. The implementation calls the clock twice: + // Once at the start of the method prior to taking the lock, when the clock will return the start time of the test. + // On the second call to the clock, where the implementation checks if the timeout has expired, the clock will + // return start_time + timeout + 1. This will cause the delete to timeout and the container will not be deleted. + kvHandler.deleteContainer(containerSet.getContainer(containerID), true); + assertEquals(1, icrReceived.get()); + assertNotNull(containerSet.getContainer(containerID)); + + // Delete the container normally, and it should go through. At this stage all calls to the clock mock will return + // the same value, indicating no delay to the delete operation will succeed. + kvHandler.deleteContainer(containerSet.getContainer(containerID), true); + assertEquals(2, icrReceived.get()); + assertNull(containerSet.getContainer(containerID)); + } + private static ContainerCommandRequestProto createContainerRequest( String datanodeId, long containerID) { return ContainerCommandRequestProto.newBuilder() diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java index 8807f22026,d2fd394271..9333ba999e --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/impl/TestFilePerBlockStrategy.java @@@ -24,36 -23,17 +24,37 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.ChunkBuffer; + import org.apache.hadoop.ozone.common.ChunkBufferToByteString; import org.apache.hadoop.ozone.container.ContainerTestHelper; +import org.apache.hadoop.ozone.container.common.ContainerTestUtils; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; +import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.common.interfaces.DBHandle; +import org.apache.hadoop.ozone.container.common.volume.MutableVolumeSet; +import org.apache.hadoop.ozone.container.common.volume.StorageVolume; import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import java.io.IOException; import java.nio.ByteBuffer; import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.stream.Stream; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.container.ContainerTestHelper.getChunk; import static org.apache.hadoop.ozone.container.ContainerTestHelper.setDataChecksum; import static org.apache.hadoop.ozone.container.common.ContainerTestUtils.WRITE_STAGE; @@@ -144,151 -124,14 +145,149 @@@ public class TestFilePerBlockStrategy e // data will be ChunkBufferImplWithByteBuffer and readData will return // ChunkBufferImplWithByteBufferList. Hence, convert both ByteStrings // before comparing. - assertEquals(data.rewind().toByteString(), - readData.rewind().toByteString()); + assertEquals(data.rewind().toByteString(), readData.toByteString()); ChunkInfo info2 = getChunk(blockID.getLocalID(), 0, start, length); - ChunkBuffer readData2 = subject.readChunk(container, blockID, info2, null); + final ChunkBufferToByteString readData2 = subject.readChunk(container, blockID, info2, null); assertEquals(length, info2.getLen()); - assertEquals(data.rewind().toByteString().substring(start, start + length), - readData2.rewind().toByteString()); + assertEquals(data.rewind().toByteString().substring(start, start + length), readData2.toByteString()); } + @ParameterizedTest + @MethodSource("getNonClosedStates") + public void testWriteChunkAndPutBlockFailureForNonClosedContainer( + ContainerProtos.ContainerDataProto.State state) throws IOException { + KeyValueContainer keyValueContainer = getKeyValueContainer(); + keyValueContainer.getContainerData().setState(state); + ContainerSet containerSet = new ContainerSet(100); + containerSet.addContainer(keyValueContainer); + KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet); + ChunkBuffer.wrap(getData()); + Assertions.assertThrows(IOException.class, () -> keyValueHandler.writeChunkForClosedContainer( + getChunkInfo(), getBlockID(), ChunkBuffer.wrap(getData()), keyValueContainer)); + Assertions.assertThrows(IOException.class, () -> keyValueHandler.putBlockForClosedContainer( + null, keyValueContainer, new BlockData(getBlockID()), 0L)); + } + + @Test + public void testWriteChunkForClosedContainer() + throws IOException { + ChunkBuffer writeChunkData = ChunkBuffer.wrap(getData()); + KeyValueContainer kvContainer = getKeyValueContainer(); + KeyValueContainerData containerData = kvContainer.getContainerData(); + closedKeyValueContainer(); + ContainerSet containerSet = new ContainerSet(100); + containerSet.addContainer(kvContainer); + KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet); + keyValueHandler.writeChunkForClosedContainer(getChunkInfo(), getBlockID(), writeChunkData, kvContainer); - ChunkBuffer readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer, ++ ChunkBufferToByteString readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer, + getBlockID(), getChunkInfo(), WRITE_STAGE); + rewindBufferToDataStart(); + Assertions.assertEquals(writeChunkData, readChunkData); + Assertions.assertEquals(containerData.getWriteBytes(), writeChunkData.remaining()); + Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining()); + + // Test Overwrite + keyValueHandler.writeChunkForClosedContainer(getChunkInfo(), getBlockID(), + writeChunkData, kvContainer); + readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer, + getBlockID(), getChunkInfo(), WRITE_STAGE); + rewindBufferToDataStart(); + Assertions.assertEquals(writeChunkData, readChunkData); + Assertions.assertEquals(containerData.getWriteBytes(), 2L * writeChunkData.remaining()); + // Overwrites won't increase the bytesUsed of a Container + Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining()); + + // Test new write chunk after overwrite + byte[] bytes = "testing write chunks with after overwrite".getBytes(UTF_8); + ChunkBuffer newWriteChunkData = ChunkBuffer.allocate(bytes.length).put(bytes); + newWriteChunkData.rewind(); + + // Write chunk after the previous overwrite chunk. + ChunkInfo newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID() + .getLocalID(), writeChunkData.remaining()), writeChunkData.remaining(), bytes.length); + keyValueHandler.writeChunkForClosedContainer(newChunkInfo, getBlockID(), + newWriteChunkData, kvContainer); + readChunkData = keyValueHandler.getChunkManager().readChunk(kvContainer, + getBlockID(), newChunkInfo, WRITE_STAGE); + newWriteChunkData.rewind(); + Assertions.assertEquals(newWriteChunkData, readChunkData); + Assertions.assertEquals(containerData.getWriteBytes(), 2L * writeChunkData.remaining() + + newWriteChunkData.remaining()); + Assertions.assertEquals(containerData.getBytesUsed(), writeChunkData.remaining() + newWriteChunkData.remaining()); + } + + @Test + public void testPutBlockForClosedContainer() throws IOException { + KeyValueContainer kvContainer = getKeyValueContainer(); + KeyValueContainerData containerData = kvContainer.getContainerData(); + closedKeyValueContainer(); + ContainerSet containerSet = new ContainerSet(100); + containerSet.addContainer(kvContainer); + KeyValueHandler keyValueHandler = createKeyValueHandler(containerSet); + List<ContainerProtos.ChunkInfo> chunkInfoList = new ArrayList<>(); + chunkInfoList.add(getChunkInfo().getProtoBufMessage()); + BlockData putBlockData = new BlockData(getBlockID()); + keyValueHandler.putBlockForClosedContainer(chunkInfoList, kvContainer, putBlockData, 1L); + Assertions.assertEquals(containerData.getBlockCommitSequenceId(), 1L); + Assertions.assertEquals(containerData.getBlockCount(), 1L); + + try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) { + long localID = putBlockData.getLocalID(); + BlockData getBlockData = dbHandle.getStore().getBlockDataTable() + .get(containerData.getBlockKey(localID)); + Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData)); + } + + // Add another chunk and check the put block data + ChunkInfo newChunkInfo = new ChunkInfo(String.format("%d.data.%d", getBlockID() + .getLocalID(), 1L), 0, 20L); + chunkInfoList.add(newChunkInfo.getProtoBufMessage()); + keyValueHandler.putBlockForClosedContainer(chunkInfoList, kvContainer, putBlockData, 2L); + Assertions.assertEquals(containerData.getBlockCommitSequenceId(), 2L); + Assertions.assertEquals(containerData.getBlockCount(), 1L); + + try (DBHandle dbHandle = BlockUtils.getDB(containerData, new OzoneConfiguration())) { + long localID = putBlockData.getLocalID(); + BlockData getBlockData = dbHandle.getStore().getBlockDataTable() + .get(containerData.getBlockKey(localID)); + Assertions.assertTrue(blockDataEquals(putBlockData, getBlockData)); + } + + // Put block on bcsId <= containerBcsId should be a no-op + keyValueHandler.putBlockForClosedContainer(chunkInfoList, kvContainer, putBlockData, 2L); + Assertions.assertEquals(containerData.getBlockCommitSequenceId(), 2L); + } + + private boolean blockDataEquals(BlockData putBlockData, BlockData getBlockData) { + return getBlockData.getSize() == putBlockData.getSize() && + Objects.equals(getBlockData.getBlockID(), putBlockData.getBlockID()) && + Objects.equals(getBlockData.getMetadata(), putBlockData.getMetadata()) && + Objects.equals(getBlockData.getChunks(), putBlockData.getChunks()); + } + + + private static Stream<Arguments> getNonClosedStates() { + return Stream.of( + Arguments.of(ContainerProtos.ContainerDataProto.State.OPEN), + Arguments.of(ContainerProtos.ContainerDataProto.State.RECOVERING), + Arguments.of(ContainerProtos.ContainerDataProto.State.CLOSING), + Arguments.of(ContainerProtos.ContainerDataProto.State.INVALID)); + } + + public KeyValueHandler createKeyValueHandler(ContainerSet containerSet) + throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(); + String dnUuid = UUID.randomUUID().toString(); + MutableVolumeSet volumeSet = new MutableVolumeSet(dnUuid, conf, + null, StorageVolume.VolumeType.DATA_VOLUME, null); + return ContainerTestUtils.getKeyValueHandler(conf, dnUuid, containerSet, volumeSet); + } + + public void closedKeyValueContainer() { + getKeyValueContainer().getContainerData().setState(ContainerProtos.ContainerDataProto.State.CLOSED); + } + @Override protected ContainerLayoutTestInfo getStrategy() { return ContainerLayoutTestInfo.FILE_PER_BLOCK; diff --cc hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java index 89522ded68,393d7e88f2..618e3bfcf4 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/ContainerCommands.java @@@ -44,25 -37,9 +37,10 @@@ import picocli.CommandLine.Command CreateSubcommand.class, CloseSubcommand.class, ReportSubcommand.class, - UpgradeSubcommand.class + UpgradeSubcommand.class, + ReconcileSubcommand.class }) @MetaInfServices(AdminSubcommand.class) - public class ContainerCommands implements Callable<Void>, AdminSubcommand { - - @Spec - private CommandSpec spec; - - @ParentCommand - private OzoneAdmin parent; - - @Override - public Void call() throws Exception { - GenericCli.missingSubcommand(spec); - return null; - } + public class ContainerCommands implements AdminSubcommand { - public OzoneAdmin getParent() { - return parent; - } } diff --cc hadoop-ozone/dist/src/main/smoketest/admincli/container.robot index e618ebd063,564fd1f5d6..b17973e1f3 --- a/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot +++ b/hadoop-ozone/dist/src/main/smoketest/admincli/container.robot @@@ -1,4 -1,4 +1,4 @@@ --# Licensed to the Apache Software Foundation (ASF) under one or more ++# Licensed to the Apache Software Foundation (ASF) under one or moreD # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 @@@ -113,6 -96,23 +102,17 @@@ List all containers from a particular c ${output} = Execute ozone admin container list --all --start 1 Should contain ${output} OPEN -Close container - ${container} = Execute ozone admin container list --state OPEN | jq -r 'select(.replicationConfig.replicationFactor == "THREE") | .containerID' | head -1 - Execute ozone admin container close "${container}" - ${output} = Execute ozone admin container info "${container}" - Should contain ${output} CLOS - Wait until keyword succeeds 1min 10sec Container is closed ${container} - + Incomplete command + ${output} = Execute And Ignore Error ozone admin container + Should contain ${output} Missing required subcommand + Should contain ${output} list + Should contain ${output} info + Should contain ${output} create + Should contain ${output} close + Should contain ${output} report + Should contain ${output} upgrade ++ Should contain ${output} reconcile + #List containers on unknown host # ${output} = Execute And Ignore Error ozone admin --verbose container list --scm unknown-host # Should contain ${output} Invalid host name diff --cc hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java index f229eb43bc,f641d4384b..939905f2ad --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ClosedContainerReplicator.java @@@ -25,10 -25,9 +25,10 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.protocol.proto.HddsProtos.LifeCycleState; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; import org.apache.hadoop.hdds.scm.container.ContainerInfo; - import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline; + import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; +import org.apache.hadoop.ozone.container.checksum.ContainerChecksumTreeManager; import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.Handler; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
