This is an automated email from the ASF dual-hosted git repository. sammichen pushed a commit to branch HDDS-5713 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 216cd287ae390e56e69179261f58f5c972e18928 Merge: 65a0cd0475b b63347345a2 Author: Sammi Chen <[email protected]> AuthorDate: Tue Jan 6 20:00:02 2026 +0800 Merge remote-tracking branch 'origin/master' into HDDS-5713 .github/workflows/check.yml | 22 +- .github/workflows/intermittent-test-check.yml | 14 +- .github/workflows/repeat-acceptance.yml | 3 +- hadoop-hdds/client/pom.xml | 2 - .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 13 + .../hdds/scm/storage/BlockDataStreamOutput.java | 8 +- .../hdds/scm/storage/MultipartInputStream.java | 4 + hadoop-hdds/common/pom.xml | 2 - .../java/org/apache/hadoop/hdds/HddsUtils.java | 116 ----- .../hdds/client/DefaultReplicationConfig.java | 4 - .../hdds/scm/container/ContainerChecksums.java | 87 ++++ .../hadoop/hdds/scm/container/ContainerInfo.java | 3 +- .../org/apache/hadoop/hdds/utils/db/Buffer.java | 80 +++ .../common/src/main/resources/ozone-default.xml | 18 + .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 133 ----- .../hdds/scm/container/TestContainerChecksums.java | 55 +++ hadoop-hdds/container-service/pom.xml | 1 - .../apache/hadoop/ozone/HddsDatanodeService.java | 6 +- .../container/common/helpers/ContainerUtils.java | 69 ++- .../container/common/helpers/DatanodeIdYaml.java | 7 + .../common/statemachine/DatanodeStateMachine.java | 3 +- .../common/states/datanode/InitDatanodeState.java | 6 +- .../container/common/utils/StorageVolumeUtil.java | 2 +- .../ozone/container/metadata/DatanodeTable.java | 5 - .../common/helpers/TestContainerUtils.java | 50 +- .../content/concept/S3-Gateway-Load-Balancing.md | 41 ++ hadoop-hdds/docs/content/design/ozone-sts.md | 215 +++++++++ hadoop-hdds/docs/content/feature/ErasureCoding.md | 35 ++ .../docs/content/start/ProductionDeployment.md | 2 +- hadoop-hdds/framework/pom.xml | 1 - .../apache/hadoop/hdds/utils/HddsServerUtil.java | 121 +++++ .../org/apache/hadoop/hdds/utils/db/DBStore.java | 2 +- .../hadoop/hdds/utils/db/RDBBatchOperation.java | 369 ++++---------- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 2 +- .../hdds/utils/db/RDBStoreCodecBufferIterator.java | 60 --- .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 9 - .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 10 - .../org/apache/hadoop/hdds/utils/db/Table.java | 8 - .../apache/hadoop/hdds/utils/db/TypedTable.java | 5 - .../apache/hadoop/ozone/audit/AuditMessage.java | 4 + .../hadoop/hdds/utils/db/InMemoryTestTable.java | 5 - .../org/apache/hadoop/hdds/utils/db/TestCodec.java | 15 +- .../hadoop/hdds/utils/db/TestCodecBufferCodec.java | 17 + .../hdds/utils/db/TestRDBBatchOperation.java | 129 ----- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 84 +--- .../hdds/utils/db/managed/ManagedDirectSlice.java | 11 +- .../hdds/utils/db/managed/ManagedObject.java | 4 +- .../utils/db/managed/TestManagedDirectSlice.java | 104 ++++ .../{managed => }/ManagedRawSSTFileIterator.java | 57 ++- .../db/{managed => }/ManagedRawSSTFileReader.java | 15 +- .../hdds/utils/db/{managed => }/package-info.java | 4 +- .../src/main/native/ManagedRawSSTFileIterator.cpp | 92 ++-- .../src/main/native/ManagedRawSSTFileReader.cpp | 8 +- .../hadoop/hdds/utils/TestNativeLibraryLoader.java | 2 +- .../TestManagedRawSSTFileIterator.java | 22 +- .../hdds/utils/db/ManagedSstFileIterator.java | 90 ++++ .../hadoop/hdds/utils/db/MinHeapMergeIterator.java | 4 +- .../hadoop/hdds/utils/db/RDBSstFileWriter.java | 0 .../hdds/utils/db}/SstFileSetReader.java | 62 +-- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 31 +- .../hdds/utils/db/TestMinHeapMergeIterator.java | 73 +-- .../hadoop/hdds/utils/db/TestRDBSstFileWriter.java | 93 ++++ .../hdds/utils/db}/TestSstFileSetReader.java | 8 +- .../apache/ozone/rocksdiff/TestCompactionDag.java | 2 +- .../rocksdiff/TestRocksDBCheckpointDiffer.java | 69 +-- hadoop-hdds/server-scm/pom.xml | 1 - .../container/AbstractContainerReportHandler.java | 2 +- .../hdds/scm/container/ContainerManagerImpl.java | 6 +- .../hdds/scm/container/ContainerReplica.java | 24 +- .../hdds/scm/container/ContainerStateManager.java | 9 +- .../scm/container/ContainerStateManagerImpl.java | 26 +- .../container/balancer/ContainerBalancerTask.java | 2 +- .../scm/container/placement/metrics/NodeStat.java | 8 +- .../container/placement/metrics/SCMNodeMetric.java | 6 +- .../container/placement/metrics/SCMNodeStat.java | 36 +- .../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 2 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 96 ++-- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 11 +- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 137 ++++++ .../hadoop/hdds/scm/container/MockNodeManager.java | 2 +- .../scm/container/TestContainerReportHandler.java | 4 +- .../scm/container/TestContainerStateManager.java | 41 ++ .../TestIncrementalContainerReportHandler.java | 4 +- .../balancer/TestContainerBalancerTask.java | 2 +- .../container/balancer/TestFindTargetStrategy.java | 22 +- .../scm/container/balancer/TestableCluster.java | 2 +- .../TestSCMContainerPlacementCapacity.java | 8 +- .../replication/TestReplicationManagerUtil.java | 6 +- .../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 2 +- .../TestCapacityPipelineChoosePolicy.java | 8 +- .../container/placement/TestDatanodeMetrics.java | 8 +- .../java/org/apache/ozone/test/OzoneTestBase.java | 16 + hadoop-ozone/cli-admin/pom.xml | 2 - hadoop-ozone/cli-shell/pom.xml | 2 - .../hadoop/ozone/shell/acl/GetAclHandler.java | 47 +- .../hadoop/ozone/shell/acl/TestGetAclHandler.java | 225 +++++++++ hadoop-ozone/client/pom.xml | 5 + .../hadoop/ozone/client/io/ECKeyOutputStream.java | 13 + .../ozone/client/io/KeyDataStreamOutput.java | 12 + .../hadoop/ozone/client/io/KeyOutputStream.java | 10 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 17 +- .../hadoop/ozone/client/OzoneClientTestUtils.java | 56 +++ hadoop-ozone/common/pom.xml | 2 - .../main/java/org/apache/hadoop/ozone/OFSPath.java | 16 +- .../hadoop/ozone/om/helpers/AclListBuilder.java | 13 + .../ozone/om/helpers/BucketEncryptionKeyInfo.java | 26 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 12 +- .../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 147 +++--- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 55 ++- .../ozone/om/helpers/WithParentObjectId.java | 14 +- .../apache/hadoop/ozone/om/helpers/WithTags.java | 2 - .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 4 +- hadoop-ozone/csi/pom.xml | 1 - .../dist/src/main/compose/ozone/docker-config | 2 +- .../src/main/smoketest/s3/presigned_url_helper.py | 73 +++ .../dist/src/main/smoketest/s3/presignedurl.robot | 54 +++ hadoop-ozone/freon/pom.xml | 2 + .../hadoop/ozone/freon/DatanodeSimulator.java | 6 +- hadoop-ozone/insight/pom.xml | 4 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 6 +- .../recon/TestStorageDistributionEndpoint.java | 206 ++++++-- hadoop-ozone/integration-test-s3/pom.xml | 11 + .../hadoop/ozone/s3/MultiS3GatewayService.java | 5 +- .../apache/hadoop/ozone/s3/S3GatewayService.java | 2 + .../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 51 ++ .../awssdk/{v1/TestS3SDKV1.java => TestS3SDK.java} | 26 +- ...aming.java => TestS3SDKWithRatisStreaming.java} | 29 +- .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 91 ++-- .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 87 ++-- .../hadoop/ozone/s3/awssdk/v2/TestS3SDKV2.java | 42 -- .../awssdk/v2/TestS3SDKV2WithRatisStreaming.java | 50 -- hadoop-ozone/integration-test/pom.xml | 6 + .../fs/ozone/AbstractOzoneFileSystemTest.java | 9 + .../ozone/AbstractRootedOzoneFileSystemTest.java | 21 + .../hadoop/fs/ozone/OzoneFileSystemTests.java | 37 ++ .../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 7 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 24 +- .../ozone/client/rpc/OzoneRpcClientTests.java | 15 +- .../TestOzoneRpcClientWithKeyLatestVersion.java | 11 +- .../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +- .../ozone/client/rpc/read/TestStreamRead.java | 201 +++++--- ...tBackgroundContainerDataScannerIntegration.java | 18 +- .../hadoop/ozone/om/snapshot/TestOmSnapshot.java | 13 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 16 +- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 7 +- .../hadoop/ozone/om/SstFilteringService.java | 33 ++ .../om/request/bucket/OMBucketCreateRequest.java | 11 +- .../ozone/om/request/file/OMFileRequest.java | 31 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 13 +- .../S3MultipartUploadCompleteRequest.java | 25 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 7 - .../s3/tagging/S3DeleteObjectTaggingRequest.java | 3 +- .../S3DeleteObjectTaggingRequestWithFSO.java | 5 +- .../s3/tagging/S3PutObjectTaggingRequest.java | 5 +- .../tagging/S3PutObjectTaggingRequestWithFSO.java | 5 +- .../om/snapshot/OmSnapshotLocalDataManager.java | 8 + .../ozone/om/snapshot/SnapshotDiffManager.java | 4 +- .../om/snapshot/defrag/SnapshotDefragService.java | 4 +- .../hadoop/ozone/om/TestBucketManagerImpl.java | 2 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 40 +- .../hadoop/ozone/om/TestOmSnapshotManager.java | 8 +- .../ozone/om/request/OMRequestTestUtils.java | 2 +- .../request/TestOMClientRequestWithUserInfo.java | 6 + .../request/bucket/TestOMBucketCreateRequest.java | 40 +- .../bucket/TestOMBucketCreateRequestWithFSO.java | 33 ++ .../TestOMDirectoriesPurgeRequestAndResponse.java | 8 +- .../om/request/key/TestOMKeyCommitRequest.java | 13 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 11 +- .../om/request/key/TestOMKeyRenameRequest.java | 6 +- .../request/key/TestOMKeyRenameRequestWithFSO.java | 20 +- .../key/TestOMKeyRenameResponseWithFSO.java | 4 +- .../response/key/TestOMOpenKeysDeleteResponse.java | 8 +- ...TestS3ExpiredMultipartUploadsAbortResponse.java | 12 +- .../tagging/TestS3DeleteObjectTaggingResponse.java | 5 +- .../s3/tagging/TestS3PutObjectTaggingResponse.java | 4 +- .../om/service/TestDirectoryDeletingService.java | 2 +- .../ozone/om/service/TestKeyDeletingService.java | 4 - .../ozone/om/snapshot/TestSnapshotDiffManager.java | 2 +- .../ozone/om/snapshot/TestSstFilteringService.java | 50 ++ .../snapshot/defrag/TestSnapshotDefragService.java | 2 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 3 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +- hadoop-ozone/recon/pom.xml | 4 +- .../ozone/recon/MetricsServiceProviderFactory.java | 30 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 3 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 12 +- .../org/apache/hadoop/ozone/recon/ReconUtils.java | 30 ++ .../ozone/recon/api/DataNodeMetricsService.java | 353 ++++++++++++++ .../ozone/recon/api/PendingDeletionEndpoint.java | 110 +++++ .../ozone/recon/api/ReconGlobalMetricsService.java | 31 +- .../recon/api/StorageDistributionEndpoint.java | 2 + .../api/types/DataNodeMetricsServiceResponse.java | 130 +++++ .../api/types/DatanodePendingDeletionMetrics.java | 59 +++ .../recon/api/types/DatanodeStorageReport.java | 22 + .../ozone/recon/api/types/ReconBasicOmKeyInfo.java | 9 +- .../ozone/recon/api/types/ScmPendingDeletion.java | 58 +++ .../ozone/recon/fsck/ContainerHealthStatus.java | 4 +- .../ozone/recon/fsck/ContainerHealthTask.java | 8 +- .../ozone/recon/scm/ContainerReplicaHistory.java | 23 +- .../ozone/recon/scm/ReconContainerManager.java | 19 +- .../ozone/recon/spi/MetricsServiceProvider.java | 5 +- .../recon/spi/impl/JmxServiceProviderImpl.java | 131 +++++ .../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +- .../spi/impl/PrometheusServiceProviderImpl.java | 39 +- .../recon/tasks/ContainerKeyMapperHelper.java | 2 +- .../recon/tasks/DataNodeMetricsCollectionTask.java | 87 ++++ .../ozone/recon/tasks/FileSizeCountTaskHelper.java | 2 +- .../recon/tasks/NSSummaryTaskDbEventHandler.java | 14 +- .../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 10 +- .../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 58 ++- .../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 20 +- .../ozone/recon/tasks/OmTableInsightTask.java | 2 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 2 +- .../ozone/recon/api/TestContainerEndpoint.java | 23 +- .../recon/api/TestNSSummaryEndpointWithFSO.java | 2 +- .../recon/api/TestNSSummaryEndpointWithLegacy.java | 2 +- .../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 2 +- .../recon/fsck/TestContainerHealthStatus.java | 9 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 5 +- .../TestContainerHealthTaskRecordGenerator.java | 5 +- .../ozone/recon/scm/TestReconContainerManager.java | 7 +- .../TestReconContainerMetadataManagerImpl.java | 18 +- .../impl/TestReconNamespaceSummaryManagerImpl.java | 2 +- .../recon/tasks/AbstractNSSummaryTaskTest.java | 4 +- hadoop-ozone/s3gateway/pom.xml | 6 + .../hadoop/ozone/s3/MultiDigestInputStream.java | 243 ++++++++++ .../ozone/s3/commontypes/RequestParameters.java | 90 ++++ .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 172 +++---- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 101 ++-- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 359 +++++++------- .../ozone/s3/endpoint/ObjectEndpointStreaming.java | 42 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 10 +- .../hadoop/ozone/s3/exception/OS3Exception.java | 2 +- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 4 + .../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 2 + .../ozone/s3/signature/StringToSignProducer.java | 2 +- .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 26 + .../hadoop/ozone/client/ObjectStoreStub.java | 3 +- .../ozone/s3/TestMultiDigestInputStream.java | 196 ++++++++ .../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 6 +- .../ozone/s3/endpoint/BucketEndpointBuilder.java | 38 -- .../hadoop/ozone/s3/endpoint/EndpointBuilder.java | 27 +- .../ozone/s3/endpoint/EndpointTestUtils.java | 189 ++++++++ .../ozone/s3/endpoint/ObjectEndpointBuilder.java | 41 -- .../s3/endpoint/TestAbortMultipartUpload.java | 13 +- .../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 28 +- .../hadoop/ozone/s3/endpoint/TestBucketList.java | 287 ++++------- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 8 +- .../s3/endpoint/TestInitiateMultipartUpload.java | 27 +- .../hadoop/ozone/s3/endpoint/TestListParts.java | 34 +- .../s3/endpoint/TestMultipartUploadComplete.java | 136 +----- .../s3/endpoint/TestMultipartUploadWithCopy.java | 134 ++--- .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 59 +-- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 537 ++++++--------------- .../ozone/s3/endpoint/TestObjectTaggingDelete.java | 59 +-- .../ozone/s3/endpoint/TestObjectTaggingGet.java | 50 +- .../ozone/s3/endpoint/TestObjectTaggingPut.java | 158 ++---- .../hadoop/ozone/s3/endpoint/TestPartUpload.java | 210 +++----- .../s3/endpoint/TestPartUploadWithStream.java | 147 ------ .../ozone/s3/endpoint/TestPermissionCheck.java | 49 +- .../ozone/s3/endpoint/TestUploadWithStream.java | 28 +- .../ozone/s3/metrics/TestS3GatewayMetrics.java | 304 ++++-------- hadoop-ozone/tools/pom.xml | 2 + .../datanode/schemaupgrade/UpgradeUtils.java | 2 +- .../hadoop/ozone/repair/om/FSORepairTool.java | 16 +- pom.xml | 21 +- 266 files changed, 6395 insertions(+), 4181 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
