This is an automated email from the ASF dual-hosted git repository. ChenSammi pushed a commit to branch HDDS-8342 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit f364114d1124d146d96689873824be5cd6b0981c Merge: f3c6ba4afae d1dbbe46574 Author: Sammi Chen <[email protected]> AuthorDate: Mon May 11 14:39:30 2026 +0800 Merge remote-tracking branch 'origin/master' into HDDS-8342 .devcontainer/devcontainer.json | 29 + .github/workflows/build-ratis.yml | 32 +- .github/workflows/check.yml | 32 +- .github/workflows/ci-with-ratis.yml | 9 +- .github/workflows/ci.yml | 67 +- .github/workflows/close-stale-prs.yaml | 4 +- .github/workflows/generate-config-doc.yml | 11 +- .github/workflows/intermittent-test-check.yml | 64 +- .github/workflows/label-pr.yml | 2 +- .github/workflows/populate-cache.yml | 14 +- .github/workflows/post-commit.yml | 7 +- .github/workflows/pull-request.yml | 6 +- .github/workflows/repeat-acceptance.yml | 22 +- .github/workflows/schedule-label-pr.yml | 5 +- .github/workflows/scheduled-cache-update.yml | 3 +- .github/workflows/update-ozone-site-config-doc.yml | 45 +- .github/workflows/{pull-request.yml => zizmor.yml} | 34 +- .mvn/extensions.xml | 2 +- CONTRIBUTING.md | 2 +- README.md | 2 +- dev-support/ci/xml_to_md.py | 9 +- dev-support/pmd/pmd-ruleset.xml | 2 + .../org/apache/hadoop/hdds/cli/GenericCli.java | 33 +- .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 33 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 3 + .../hadoop/hdds/scm/XceiverClientManager.java | 11 +- .../io/ECBlockReconstructedStripeInputStream.java | 8 +- .../common/dev-support/findbugsExcludeFile.xml | 5 - .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 22 + .../hadoop/hdds/protocol/DatanodeDetails.java | 2 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 10 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 + .../hadoop/hdds/security/SecurityConfig.java | 36 + .../org/apache/hadoop/ipc_/AlignmentContext.java | 2 +- .../src/main/java/org/apache/hadoop/ipc_/RPC.java | 18 +- .../main/java/org/apache/hadoop/ipc_/Server.java | 51 +- .../org/apache/hadoop/ipc_/WritableRpcEngine.java | 630 ----- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 10 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +- .../hadoop/ozone/common/ChecksumByteBuffer.java | 78 - .../ozone/common/PureJavaCrc32ByteBuffer.java | 23 +- .../ozone/common/PureJavaCrc32CByteBuffer.java | 23 +- .../security_/CustomizedCallbackHandler.java | 121 + .../hadoop/security_/SaslMechanismFactory.java | 90 + .../org/apache/hadoop/security_/SaslRpcClient.java | 16 +- .../org/apache/hadoop/security_/SaslRpcServer.java | 63 +- .../common/src/main/resources/ozone-default.xml | 174 +- .../security/TestSecurityConfigTlsSettings.java | 90 + .../ozone/common/TestChecksumByteBuffer.java | 4 +- .../common/TestChecksumImplsComputeSameValues.java | 4 +- .../ozone/container/common/impl/ContainerData.java | 11 +- .../common/statemachine/DatanodeConfiguration.java | 138 + .../commandhandler/DeleteBlocksCommandHandler.java | 6 +- .../common/transport/server/XceiverServerGrpc.java | 13 +- .../server/ratis/ContainerStateMachine.java | 14 +- .../ozone/container/common/volume/HddsVolume.java | 2 +- .../container/common/volume/StorageVolume.java | 44 + .../common/volume/StorageVolumeChecker.java | 114 +- .../diskbalancer/DiskBalancerConfiguration.java | 120 +- .../container/diskbalancer/DiskBalancerInfo.java | 29 +- .../diskbalancer/DiskBalancerProtocolServer.java | 3 +- .../diskbalancer/DiskBalancerService.java | 53 +- .../container/diskbalancer/DiskBalancerYaml.java | 20 +- .../policy/ContainerChoosingPolicy.java | 5 +- .../policy/DefaultContainerChoosingPolicy.java | 97 +- .../ozone/container/metadata/AbstractRDBStore.java | 4 +- .../container/replication/ReplicationServer.java | 5 + .../statemachine/TestDatanodeConfiguration.java | 32 + .../server/ratis/TestContainerStateMachine.java | 45 +- .../common/volume/TestStorageVolumeChecker.java | 185 ++ .../volume/TestStorageVolumeHealthChecks.java | 94 + .../TestDefaultContainerChoosingPolicy.java | 82 +- .../TestDiskBalancerProtocolServer.java | 4 + .../diskbalancer/TestDiskBalancerService.java | 61 +- .../diskbalancer/TestDiskBalancerTask.java | 12 +- .../diskbalancer/TestDiskBalancerYaml.java | 79 +- hadoop-hdds/docs/content/design/diskbalancer.md | 4 +- .../docs/content/design/s3-conditional-requests.md | 668 ++++- hadoop-hdds/docs/content/feature/DiskBalancer.md | 42 +- .../docs/content/feature/DiskBalancer.zh.md | 4 +- .../feature/Snapshot-Configuration-Properties.md | 1 + hadoop-hdds/docs/content/feature/Snapshot.md | 9 +- .../content/feature/SnapshotDefragmentation.md | 212 +- hadoop-hdds/framework/pom.xml | 4 + .../hadoop/hdds/fs/CachingSpaceUsageSource.java | 12 +- .../hadoop/hdds/scm/client/ScmTopologyClient.java | 2 +- ...lockLocationProtocolClientSideTranslatorPB.java | 8 + .../symmetric/DefaultSecretKeySignerClient.java | 2 +- .../server/OzoneProtocolMessageDispatcher.java | 3 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 28 +- .../hadoop/hdds/server/http/HttpServer2.java | 33 +- .../org/apache/hadoop/hdds/utils/Archiver.java | 38 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 15 +- .../hadoop/hdds/utils/db/RDBCheckpointManager.java | 14 +- .../org/apache/hadoop/ozone/common/BlockGroup.java | 9 +- .../apache/hadoop/ozone/common/DeletedBlock.java | 11 +- .../resources/webapps/static/templates/menu.html | 3 +- .../hdds/fs/TestCachingSpaceUsageSource.java | 35 + .../hdds/security/ssl/TestGrpcTlsConfig.java | 276 ++ .../hdds/server/http/TestHttpServer2SSL.java | 361 +++ .../org/apache/hadoop/hdds/utils/TestArchiver.java | 18 +- .../src/main/proto/RpcHeader.proto | 2 +- .../interface-client/src/main/proto/hdds.proto | 2 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 + .../src/main/proto/ScmServerProtocol.proto | 1 + .../rocksdiff/TestRocksDBCheckpointDiffer.java | 2 + .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 5 + .../hdds/scm/container/ContainerManagerImpl.java | 12 +- .../hdds/scm/container/ContainerStateManager.java | 5 + .../scm/container/ContainerStateManagerImpl.java | 3 +- .../algorithms/SCMContainerPlacementRackAware.java | 2 - .../container/placement/metrics/SCMMetrics.java | 36 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 5 + .../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 14 +- .../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 2 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 7 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 43 +- .../ha/invoker/ContainerStateManagerInvoker.java | 328 +++ .../DeletedBlockLogStateManagerInvoker.java | 20 +- .../ha/invoker/PipelineStateManagerInvoker.java | 282 ++ .../hadoop/hdds/scm/ha/invoker/ScmInvoker.java | 3 +- .../scm/ha/invoker/ScmInvokerCodeGenerator.java | 251 +- .../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 17 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 13 + .../hadoop/hdds/scm/node/NodeStateManager.java | 4 +- .../hdds/scm/node/PendingContainerTracker.java | 225 ++ .../hadoop/hdds/scm/node/SCMNodeManager.java | 30 + .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 24 + .../hadoop/hdds/scm/pipeline/PipelineManager.java | 21 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 17 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 2 - .../hdds/scm/pipeline/PipelineStateManager.java | 5 + .../scm/pipeline/PipelineStateManagerImpl.java | 3 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 3 +- .../hdds/scm/server/SCMClientProtocolServer.java | 12 + .../hdds/scm/server/SCMHTTPServerConfig.java | 20 +- .../hdds/scm/server/StorageContainerManager.java | 16 +- .../server/StorageContainerManagerHttpServer.java | 2 +- .../main/resources/webapps/scm/ratis-events.html} | 36 +- .../src/main/resources/webapps/scm/scm.js | 26 + .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 1 + .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 10 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 37 +- .../hdds/scm/container/SimpleMockNodeManager.java | 12 +- .../scm/container/TestContainerManagerImpl.java | 7 +- .../algorithms/TestContainerPlacementFactory.java | 2 +- .../TestSCMContainerPlacementCapacity.java | 3 +- .../TestSCMContainerPlacementRackAware.java | 6 +- .../TestSCMContainerPlacementRackScatter.java | 6 +- .../TestSCMContainerPlacementRandom.java | 6 +- .../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 52 + .../hdds/scm/node/TestContainerPlacement.java | 3 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 2 + .../hdds/scm/node/TestPendingContainerTracker.java | 381 +++ .../hdds/scm/node/states/TestNodeStateMap.java | 3 +- .../hdds/scm/pipeline/MockPipelineManager.java | 11 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 56 +- .../scm/pipeline/TestPipelinePlacementFactory.java | 3 +- .../scm/cli/ContainerBalancerStatusSubcommand.java | 19 +- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 5 +- .../hdds/scm/cli/container/InfoSubcommand.java | 12 +- .../scm/cli/container/ReconcileSubcommand.java | 26 +- .../hdds/scm/cli/container/ReportSubcommand.java | 10 +- .../hdds/scm/cli/datanode/BasicDatanodeInfo.java | 54 +- .../cli/datanode/DecommissionStatusSubCommand.java | 6 +- .../scm/cli/datanode/DecommissionSubCommand.java | 4 +- .../cli/datanode/DiskBalancerReportSubcommand.java | 10 +- .../cli/datanode/DiskBalancerStartSubcommand.java | 11 + .../cli/datanode/DiskBalancerStatusSubcommand.java | 13 +- .../cli/datanode/DiskBalancerUpdateSubcommand.java | 15 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 62 +- .../scm/cli/datanode/MaintenanceSubCommand.java | 4 +- .../scm/cli/datanode/RecommissionSubCommand.java | 4 +- .../ozone/admin/nssummary/DiskUsageSubCommand.java | 11 +- .../admin/nssummary/FileSizeDistSubCommand.java | 4 +- .../ozone/admin/nssummary/NSSummaryCLIUtils.java | 11 +- .../ozone/admin/om/ListOpenFilesSubCommand.java | 55 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 18 +- .../ozone/admin/scm/RotateKeySubCommand.java | 2 +- .../hdds/scm/cli/container/TestInfoSubCommand.java | 5 +- .../scm/cli/container/TestReconcileSubcommand.java | 50 +- .../cli/datanode/TestDiskBalancerSubCommands.java | 18 + .../scm/cli/datanode/TestListInfoSubcommand.java | 60 + hadoop-ozone/cli-debug/pom.xml | 4 + .../ozone/debug/kerberos/AuthorizationProbe.java | 106 + .../hadoop/ozone/debug/kerberos/ConfigProbe.java | 116 + .../ozone/debug/kerberos/DiagnoseSubcommand.java | 113 + .../ozone/debug/kerberos/DiagnosticProbe.java | 36 + .../ozone/debug/kerberos/EnvironmentProbe.java | 43 + .../hadoop/ozone/debug/kerberos/HostProbe.java | 72 + .../hadoop/ozone/debug/kerberos/HttpAuthProbe.java | 52 + .../ozone/debug/kerberos/JvmKerberosProbe.java | 98 + .../ozone/debug/kerberos/KerberosConfigProbe.java | 61 + .../ozone/debug/kerberos/KerberosSubcommand.java | 36 + .../ozone/debug/kerberos/KerberosTicketProbe.java | 95 + .../hadoop/ozone/debug/kerberos/KeytabProbe.java | 93 + .../hadoop/ozone/debug/kerberos/KinitProbe.java | 61 + .../debug/kerberos/PrincipalMappingProbe.java | 113 + .../hadoop/ozone/debug/kerberos/ProbeResult.java | 27 + .../ozone/debug/kerberos/SecurityConfigProbe.java | 72 + .../kerberos/TranslatePrincipalSubcommand.java | 78 + .../hadoop/ozone/debug/kerberos/package-info.java | 21 + .../debug/kerberos/TestOzoneDebugKerberos.java | 159 ++ .../hadoop/ozone/debug/kerberos/package-info.java | 21 + .../hadoop/ozone/shell/OzoneInteractiveShell.java | 84 + .../hadoop/ozone/shell/keys/ListKeyHandler.java | 10 +- .../ozone/shell/snapshot/SnapshotDiffHandler.java | 41 +- .../apache/hadoop/ozone/client/ObjectStore.java | 46 + .../apache/hadoop/ozone/client/OzoneBucket.java | 21 + .../ozone/client/protocol/ClientProtocol.java | 58 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 71 +- .../hadoop/ozone/client/TestOzoneECClient.java | 9 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 34 +- .../hadoop/ozone/client/checksum/CrcComposer.java | 79 +- .../hadoop/ozone/client/checksum/CrcUtil.java | 155 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 - .../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 21 + .../hadoop/ozone/om/helpers/SnapshotDiffJob.java | 9 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 50 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 96 +- .../ozone/security/acl/AssumeRoleRequest.java | 31 +- .../hadoop/ozone/security/acl/RequestContext.java | 31 + .../ozone/snapshot/SnapshotDiffResponse.java | 59 +- .../ozone/snapshot/SubmitSnapshotDiffResponse.java | 65 + .../ozone/client/checksum/TestCrcComposer.java | 222 ++ .../hadoop/ozone/client/checksum/TestCrcUtil.java | 236 ++ .../hadoop/ozone/om/helpers/TestQuotaUtil.java | 24 + .../ozone/security/acl/TestAssumeRoleRequest.java | 73 +- .../hadoop/ozone/security/acl/package-info.java | 21 + .../ozone/snapshot/TestSnapshotDiffResponse.java | 78 + .../snapshot/TestSubmitSnapshotDiffResponse.java | 97 + hadoop-ozone/dist/README.md | 6 +- hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/assemblies/ozone-src.xml | 7 + .../compose/ozonesecure-ha/test-debug-tools.sh | 2 + .../dist/src/main/compose/xcompat/new-cluster.yaml | 3 + .../dist/src/main/compose/xcompat/old-cluster.yaml | 3 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 1 + hadoop-ozone/dist/src/main/license/jar-report.txt | 1 + .../src/main/smoketest/compatibility/setup.robot | 9 +- .../src/main/smoketest/compatibility/write.robot | 65 + .../smoketest/debug/ozone-debug-kerberos.robot | 59 + .../omha/data-validation-after-om-bootstrap.robot | 15 +- .../main/smoketest/scmha/container-create.robot | 2 +- .../src/main/smoketest/snapshot/snapshot-sh.robot | 10 +- hadoop-ozone/dist/src/shell/ozone/ozone | 12 + .../src/main/resources/httpfs-default.xml | 8 - hadoop-ozone/iceberg/pom.xml | 26 +- .../ozone/iceberg/RewriteTablePathOzoneAction.java | 279 +- .../ozone/iceberg/RewriteTablePathOzoneUtils.java | 126 + .../apache/hadoop/ozone/iceberg/package-info.java | 21 + .../iceberg/TestRewriteTablePathOzoneAction.java | 279 ++ ...> AbstractTestStorageDistributionEndpoint.java} | 280 +- .../ozone/recon/TestReconAndAdminContainerCLI.java | 1 + .../ozone/recon/TestReconContainerEndpoint.java | 71 +- .../ozone/recon/TestReconOmMetaManagerUtils.java | 33 + .../ozone/recon/TestReconWithOzoneManagerHA.java | 2 + .../recon/TestStorageDistributionEndpointEC.java | 93 + .../TestStorageDistributionEndpointRatis.java | 100 + hadoop-ozone/integration-test-s3/pom.xml | 2 +- .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 303 +++ .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 434 +++ .../fs/ozone/AbstractOzoneFileSystemTest.java | 147 +- .../ozone/AbstractRootedOzoneFileSystemTest.java | 110 +- .../hadoop/fs/ozone/OzoneFileSystemTestBase.java | 114 + .../hadoop/fs/ozone/TestOzoneFSBucketLayout.java | 5 +- .../hadoop/fs/ozone/TestOzoneFsSnapshot.java | 51 + .../hdds/scm/TestSCMDbCheckpointServlet.java | 2 +- .../hdds/scm/TestStorageContainerManager.java | 11 +- .../hadoop/hdds/scm/TestXceiverClientGrpc.java | 27 + .../TestScmDataDistributionFinalization.java | 6 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 5 - .../ozone/client/rpc/OzoneRpcClientTests.java | 164 +- .../rpc/TestContainerStateMachineFailures.java | 65 +- .../replication/TestContainerReplication.java | 53 + .../TestContainerCommandReconciliation.java | 3 +- .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 1 + .../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 + .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 1 + .../ozone/om/service/TestRangerBGSyncService.java | 2 +- .../om/snapshot/TestOzoneManagerHASnapshot.java | 2 + .../om/snapshot/TestOzoneManagerSnapshotAcl.java | 2 + .../node/TestDiskBalancerPolicyPerformance.java | 13 +- .../ozone/shell/TestDeletedBlocksTxnShell.java | 3 +- .../src/main/proto/OmClientProtocol.proto | 53 +- .../org/apache/hadoop/ozone/audit/OMAction.java | 1 + .../hadoop/ozone/om/GrpcOzoneManagerServer.java | 5 + .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 21 + .../om/OMDBCheckpointServletInodeBasedXfer.java | 14 + .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 33 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 9 +- .../hadoop/ozone/om/OmSnapshotLocalData.java | 2 +- .../hadoop/ozone/om/OmSnapshotLocalDataYaml.java | 15 +- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 59 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 159 +- .../hadoop/ozone/om/OzoneManagerHttpServer.java | 2 + .../java/org/apache/hadoop/ozone/om/ScmClient.java | 57 +- .../hadoop/ozone/om/SnapshotListJSONServlet.java | 96 + .../ozone/om/ratis/OzoneManagerStateMachine.java | 49 +- .../om/ratis_snapshot/OmRatisSnapshotProvider.java | 2 +- .../ozone/om/request/file/OMFileRequest.java | 6 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 5 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 1 - .../ozone/om/request/key/OMKeyCreateRequest.java | 55 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 96 +- .../S3MultipartUploadCompleteRequest.java | 7 + .../request/snapshot/OMSnapshotRenameRequest.java | 10 + .../ozone/om/service/DirectoryDeletingService.java | 30 +- .../ozone/om/service/KeyDeletingService.java | 8 +- .../om/snapshot/OmSnapshotLocalDataManager.java | 25 +- .../ozone/om/snapshot/SnapshotDiffManager.java | 179 +- .../protocolPB/OzoneManagerRequestHandler.java | 62 +- .../webapps/ozoneManager/om-snapshots.html | 52 + .../resources/webapps/ozoneManager/ozoneManager.js | 53 + .../webapps/ozoneManager/ratis-events.html | 36 +- ...DBCheckpointServletInodeBasedXferNonLeader.java | 80 + .../ozone/om/TestOmSnapshotLocalDataYaml.java | 50 +- .../TestOzoneManagerListMultipartUploadsAcls.java | 196 ++ .../ozone/om/TestOzoneManagerListPartsAcls.java | 196 ++ .../org/apache/hadoop/ozone/om/TestScmClient.java | 55 +- .../ozone/om/TestSnapshotListJSONServlet.java | 57 + .../om/ratis/TestOzoneManagerDoubleBuffer.java | 2 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +- .../om/ratis/TestOzoneManagerStateMachine.java | 25 + .../request/TestOMClientRequestWithUserInfo.java | 2 +- .../ozone/om/request/bucket/TestBucketRequest.java | 2 +- .../request/file/TestOMDirectoryCreateRequest.java | 2 +- .../file/TestOMDirectoryCreateRequestWithFSO.java | 2 +- .../om/request/key/TestOMKeyCommitRequest.java | 4 +- .../om/request/key/TestOMKeyCreateRequest.java | 8 +- .../ozone/om/request/key/TestOMKeyRequest.java | 4 +- .../TestOMLifecycleConfigurationRequest.java | 2 +- .../s3/multipart/TestS3MultipartRequest.java | 2 +- .../s3/security/TestS3GetSecretRequest.java | 2 +- .../s3/tenant/TestOMTenantCreateRequest.java | 2 +- .../s3/tenant/TestOMTenantDeleteRequest.java | 2 +- .../snapshot/TestOMSnapshotRenameRequest.java | 20 + .../om/request/volume/TestOMVolumeRequest.java | 2 +- .../om/service/TestDirectoryDeletingService.java | 94 +- .../ozone/om/service/TestKeyDeletingService.java | 5 +- .../snapshot/TestOmSnapshotLocalDataManager.java | 10 + .../ozone/om/snapshot/TestSnapshotCache.java | 8 +- .../ozone/om/snapshot/TestSnapshotDiffManager.java | 298 ++- .../om/snapshot/TestSnapshotDiffManagerMXBean.java | 5 +- .../snapshot/TestSnapshotRequestAndResponse.java | 4 +- .../protocolPB/TestOzoneManagerRequestHandler.java | 87 + .../ozone/security/acl/TestRequestContext.java | 86 + .../ozone/BasicRootedOzoneClientAdapterImpl.java | 13 +- hadoop-ozone/ozonefs-shaded/pom.xml | 7 +- .../hadoop/ozone/recon/ReconControllerModule.java | 2 + .../hadoop/ozone/recon/ReconServerConfigKeys.java | 39 + .../hadoop/ozone/recon/api/ContainerEndpoint.java | 166 +- .../hadoop/ozone/recon/api/ExportJobManager.java | 404 +++ .../recon/api/StorageDistributionEndpoint.java | 53 +- .../ozone/recon/api/handlers/BucketHandler.java | 14 +- .../hadoop/ozone/recon/api/types/ExportJob.java | 222 ++ .../persistence/ContainerHealthSchemaManager.java | 95 +- .../webapps/recon/ozone-recon-web/api/db.json | 2799 ++++++++++++++++++-- .../recon/ozone-recon-web/api/pagination.js | 86 + .../webapps/recon/ozone-recon-web/api/routes.json | 5 + .../webapps/recon/ozone-recon-web/package.json | 2 +- .../src/v2/components/tables/containersTable.tsx | 71 +- .../src/v2/pages/containers/containers.tsx | 816 ++++-- .../src/v2/types/container.types.ts | 56 +- .../ozone/recon/api/TestExportJobManager.java | 432 +++ .../recon/api/TestStorageDistributionEndpoint.java | 12 +- .../ozone/recon/api/types/TestExportJob.java | 110 + .../recon/fsck/TestReconReplicationManager.java | 2 +- .../TestUnhealthyContainersDerbyPerformance.java | 3 +- .../hadoop/ozone/s3/AuthorizationFilter.java | 7 +- .../ozone/s3/commontypes/EncodingTypeObject.java | 5 +- .../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 84 +- .../ozone/s3/endpoint/BucketCrudHandler.java | 38 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 29 +- ...CompleteMultipartUploadRequestUnmarshaller.java | 4 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 24 - .../s3/endpoint/ListMultipartUploadsHandler.java | 5 +- .../ozone/s3/endpoint/MessageUnmarshaller.java | 2 +- .../ozone/s3/endpoint/MultipartKeyHandler.java | 2 - .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 170 +- .../ozone/s3/endpoint/ObjectEndpointStreaming.java | 8 +- .../ozone/s3/endpoint/ObjectTaggingHandler.java | 2 +- .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 86 +- .../ozone/s3/endpoint/S3ConditionalRequest.java | 74 +- .../hadoop/ozone/s3/exception/OS3Exception.java | 33 +- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 98 +- .../ozone/s3/signature/StringToSignProducer.java | 14 +- .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 4 + .../hadoop/ozone/client/ClientProtocolStub.java | 34 + .../hadoop/ozone/client/OzoneBucketStub.java | 25 + .../s3/commontypes/TestObjectKeyNameAdapter.java | 5 + .../hadoop/ozone/s3/endpoint/TestBucketList.java | 3 + .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 118 + .../hadoop/ozone/local/LocalOzoneRuntime.java | 39 + .../org/apache/hadoop/ozone/local/OzoneLocal.java | 52 + .../apache/hadoop/ozone/local/package-info.java | 21 + .../apache/hadoop/ozone/local/TestOzoneLocal.java | 100 + pom.xml | 33 +- .../FileSystem/failure_injector_fs.cc | 2 +- 402 files changed, 19261 insertions(+), 3724 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 757a09d9d67,bce241f0ded..cab32ed7387 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@@ -318,9 -318,8 +318,9 @@@ public final class OzoneConsts public static final String TENANT = "tenant"; public static final String USER_PREFIX = "userPrefix"; public static final String REWRITE_GENERATION = "rewriteGeneration"; + public static final String DELETED_KEY_SOURCE_TYPE = "deletedKeySourceType"; /** Sentinel generation used to request atomic create-if-not-exists(put if absent) semantics. */ - public static final long EXPECTED_GEN_CREATE_IF_NOT_EXISTS = -1L; + public static final long EXPECTED_GEN_CREATE_IF_ABSENT = 0L; public static final String FROM_SNAPSHOT = "fromSnapshot"; public static final String TO_SNAPSHOT = "toSnapshot"; public static final String TOKEN = "token"; diff --cc hadoop-hdds/common/src/main/resources/ozone-default.xml index 9f4e38b0739,f554b440602..a7c4d56e531 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@@ -5045,63 -5068,16 +5069,75 @@@ <value>5m</value> <description>Interval for cleaning up orphan snapshot local data versions corresponding to snapshots</description> </property> + + <property> + <name>ozone.scm.ratis.events.max.limit</name> + <value>100</value> + <tag>OZONE, RATIS, SCM</tag> + <description>The maximum number of events that can be pending in SCM Ratis.</description> + </property> + <property> + <name>ozone.om.ratis.events.max.limit</name> + <value>100</value> + <tag>OZONE, RATIS, OM</tag> + <description>The maximum number of events that can be pending in OM Ratis.</description> + </property> ++ + <property> + <name>ozone.lifecycle.service.enabled</name> + <value>false</value> + <tag>OZONE</tag> + <description>It specifies whether to enable lifecycle management service.</description> + </property> - + <property> + <name>ozone.lifecycle.service.move.to.trash.enabled</name> + <value>true</value> + <tag>OZONE</tag> + <description>When enabled KeyLifecycleService will move expired keys/dirs to trash if trash is available. + When disabled, it will delete them directly. + </description> + </property> + <property> + <name>ozone.lifecycle.service.delete.batch-size</name> + <value>1000</value> + <tag>OZONE</tag> + <description>Max numbers of objects allowed for deletion in a batch for a lifecycle evaluating task.</description> + </property> + <property> + <name>ozone.lifecycle.service.interval</name> + <value>24h</value> + <tag>OZONE</tag> + <description>Interval at which key lifecycle management service will run.</description> + </property> + <property> + <name>ozone.lifecycle.service.timeout</name> + <value>2h</value> + <tag>OZONE</tag> + <description>Timeout for task of key lifecycle management service.</description> + </property> + <property> + <name>ozone.lifecycle.service.workers</name> + <value>5</value> + <tag>OZONE</tag> + <description>Number of workers executed of key lifecycle management service. This + configuration should be set to greater than 0.</description> + </property> + <property> + <name>ozone.lifecycle.service.delete.cached.directory.max-count</name> + <value>1000000</value> + <tag>OZONE</tag> + <description>Max numbers of directory objects held in memory stack for recursive FSO bucket evaluating for + a lifecycle evaluating task. Once the cached directory objects exceeds this limit, the evaluation of the involved + directories will abort. + </description> + </property> + <property> + <name>ozone.lifecycle.service.mpu.abort.limit.per.task</name> + <value>1000</value> + <tag>OZONE</tag> + <description>Maximum number of multipart upload parts (rounded up to complete uploads) + to abort in a single task when processing AbortIncompleteMultipartUpload + lifecycle rules. + </description> + </property> </configuration> diff --cc hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index bb2da9e9fde,8238ade3921..60100d2b5ae --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@@ -43,9 -41,10 +43,11 @@@ import com.amazonaws.services.s3.model. import com.amazonaws.services.s3.model.CanonicalGrantee; import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest; import com.amazonaws.services.s3.model.CompleteMultipartUploadResult; + import com.amazonaws.services.s3.model.CopyObjectRequest; + import com.amazonaws.services.s3.model.CopyObjectResult; import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.GetBucketLifecycleConfigurationRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.Grantee; import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest; diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 19828030d64,22c71cc2985..d0a110b7263 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -156,11 -156,7 +156,12 @@@ enum Type PutObjectTagging = 140; GetObjectTagging = 141; DeleteObjectTagging = 142; - SetLifecycleConfiguration = 150; - GetLifecycleConfiguration = 151; - DeleteLifecycleConfiguration = 152; - GetLifecycleServiceStatus = 153; - SetLifecycleServiceStatus = 154; + SubmitSnapshotDiff = 143; ++ SetLifecycleConfiguration = 144; ++ GetLifecycleConfiguration = 145; ++ DeleteLifecycleConfiguration = 146; ++ GetLifecycleServiceStatus = 147; ++ SetLifecycleServiceStatus = 148; } enum SafeMode { @@@ -311,12 -307,8 +312,13 @@@ message OMRequest optional PutObjectTaggingRequest putObjectTaggingRequest = 141; optional DeleteObjectTaggingRequest deleteObjectTaggingRequest = 142; repeated SetSnapshotPropertyRequest SetSnapshotPropertyRequests = 143; ++ optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144; - optional SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = 144; - optional GetLifecycleConfigurationRequest getLifecycleConfigurationRequest = 145; - optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 146; - optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 147; - optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 148; - optional SubmitSnapshotDiffRequest submitSnapshotDiffRequest = 144; ++ optional SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = 145; ++ optional GetLifecycleConfigurationRequest getLifecycleConfigurationRequest = 146; ++ optional DeleteLifecycleConfigurationRequest deleteLifecycleConfigurationRequest = 147; ++ optional GetLifecycleServiceStatusRequest getLifecycleServiceStatusRequest = 148; ++ optional SetLifecycleServiceStatusRequest setLifecycleServiceStatusRequest = 149; } message OMResponse { @@@ -450,13 -442,8 +452,13 @@@ optional GetObjectTaggingResponse getObjectTaggingResponse = 140; optional PutObjectTaggingResponse putObjectTaggingResponse = 141; optional DeleteObjectTaggingResponse deleteObjectTaggingResponse = 142; - + optional SubmitSnapshotDiffResponse submitSnapshotDiffResponse = 143; + - optional SetLifecycleConfigurationResponse setLifecycleConfigurationResponse = 150; - optional GetLifecycleConfigurationResponse getLifecycleConfigurationResponse = 151; - optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 152; - - optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 153; - optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 154; ++ optional SetLifecycleConfigurationResponse setLifecycleConfigurationResponse = 144; ++ optional GetLifecycleConfigurationResponse getLifecycleConfigurationResponse = 145; ++ optional DeleteLifecycleConfigurationResponse deleteLifecycleConfigurationResponse = 146; ++ optional GetLifecycleServiceStatusResponse getLifecycleServiceStatusResponse = 147; ++ optional SetLifecycleServiceStatusResponse setLifecycleServiceStatusResponse = 148; } enum Status { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java index c6e0640d5ba,342bdf7c746..4cbae8ebec3 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMAction.java @@@ -118,12 -118,7 +118,13 @@@ public enum OMAction implements AuditAc GET_SNAPSHOT_DIFF_REPORT, LIST_SNAPSHOT_DIFF_JOBS, CANCEL_SNAPSHOT_DIFF_JOBS, - SUBMIT_SNAPSHOT_DIFF_JOB; ++ SUBMIT_SNAPSHOT_DIFF_JOB, + + GET_LIFECYCLE_CONFIGURATION, + SET_LIFECYCLE_CONFIGURATION, + DELETE_LIFECYCLE_CONFIGURATION, + GET_LIFECYCLE_SERVICE_STATUS, + SET_LIFECYCLE_SERVICE_STATUS; @Override public String getAction() { diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java index 7d1c81967d6,00000000000..ddc581c5029 mode 100644,000000..100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java @@@ -1,153 -1,0 +1,153 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.lifecycle; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.util.UUID; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.AuditMessage; +import org.apache.hadoop.ozone.om.OMConfigKeys; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OmMetadataManagerImpl; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.ResolvedBucket; +import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.request.OMClientRequest; +import org.apache.hadoop.ozone.om.request.OMRequestTestUtils; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.BucketLayoutProto; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DeleteLifecycleConfigurationRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleAction; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleConfiguration; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleExpiration; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleFilter; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.LifecycleRule; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.SetLifecycleConfigurationRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mockito; + +/** + * Base test class for Lifecycle configuration request. + */ +@SuppressWarnings("visibilitymodifier") +public class TestOMLifecycleConfigurationRequest { + + @TempDir + private File tempDir; + + protected OzoneManager ozoneManager; + protected OMMetrics omMetrics; + protected OMMetadataManager omMetadataManager; + protected AuditLogger auditLogger; + + @BeforeEach + public void setup() throws Exception { + ozoneManager = mock(OzoneManager.class); - omMetrics = OMMetrics.create(); + OzoneConfiguration ozoneConfiguration = new OzoneConfiguration(); ++ omMetrics = OMMetrics.create(ozoneConfiguration); + ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS, tempDir.getAbsolutePath()); + omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration, ozoneManager); + when(ozoneManager.getMetrics()).thenReturn(omMetrics); + when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); + when(ozoneManager.getMaxUserVolumeCount()).thenReturn(10L); + when(ozoneManager.resolveBucketLink(any(Pair.class), any(OMClientRequest.class))) + .thenAnswer(i -> new ResolvedBucket(i.getArgument(0), + i.getArgument(0), "dummyBucketOwner", BucketLayout.OBJECT_STORE)); + when(ozoneManager.isStrictS3()).thenReturn(true); + OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class); + when(lvm.getMetadataLayoutVersion()).thenReturn(0); + when(ozoneManager.getVersionManager()).thenReturn(lvm); + auditLogger = mock(AuditLogger.class); + when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); + Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); + } + + @AfterEach + public void stop() { + omMetrics.unRegister(); + Mockito.framework().clearInlineMocks(); + } + + public OMRequest createDeleteLifecycleConfigurationRequest( + String volumeName, String bucketName) { + return OMRequest.newBuilder().setDeleteLifecycleConfigurationRequest( + DeleteLifecycleConfigurationRequest.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName)) + .setCmdType(Type.DeleteLifecycleConfiguration) + .setClientId(UUID.randomUUID().toString()).build(); + } + + public static void addVolumeAndBucketToTable(String volumeName, + String bucketName, String ownerName, OMMetadataManager omMetadataManager) + throws Exception { + OMRequestTestUtils.addVolumeToDB(volumeName, ownerName, omMetadataManager); + OMRequestTestUtils.addBucketToDB(volumeName, bucketName, omMetadataManager); + } + + public OMRequest setLifecycleConfigurationRequest(String volumeName, + String bucketName, String ownerName) { + return setLifecycleConfigurationRequest(volumeName, bucketName, + ownerName, true); + } + + public OMRequest setLifecycleConfigurationRequest(String volumeName, + String bucketName, String ownerName, boolean addRules) { + String prefix = "prefix/"; + LifecycleConfiguration.Builder builder = LifecycleConfiguration.newBuilder() + .setBucketLayout(BucketLayoutProto.OBJECT_STORE) + .setCreationTime(System.currentTimeMillis()) + .setVolume(volumeName) + .setBucket(bucketName); + + if (addRules) { + builder.addRules(LifecycleRule.newBuilder() + .setId(RandomStringUtils.randomAlphabetic(32)) + .setEnabled(true) + .addAction(LifecycleAction.newBuilder() + .setExpiration(LifecycleExpiration.newBuilder().setDays(3).build())) + .setFilter(LifecycleFilter.newBuilder().setPrefix(prefix)) + ); + } + + LifecycleConfiguration lcc = builder.build(); + + SetLifecycleConfigurationRequest setLifecycleConfigurationRequest = + SetLifecycleConfigurationRequest.newBuilder() + .setLifecycleConfiguration(lcc) + .build(); + + return OMRequest.newBuilder().setSetLifecycleConfigurationRequest( + setLifecycleConfigurationRequest) + .setCmdType(Type.SetLifecycleConfiguration) + .setClientId(UUID.randomUUID().toString()) + .build(); + } +} diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java index 098f9136062,a2698b3098e..c91fb91421f --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketCrudHandler.java @@@ -61,21 -50,6 +61,21 @@@ public class BucketCrudHandler extends && queryParams().get(QueryParams.DELETE) == null; } + /** + * Handle GET /{bucket} for bucket LIFECYCLE configuration. + */ + @Override + Response handleGetRequest(S3RequestContext context, String bucketName) + throws IOException, OS3Exception { + final String lifecycleMarker = queryParams().get(QueryParams.LIFECYCLE); + + if (lifecycleMarker != null) { + context.setAction(S3GAction.GET_BUCKET_LIFECYCLE); - return getBucketLifecycleConfiguration(bucketName); ++ return getBucketLifecycleConfiguration(context, bucketName); + } + return null; + } + /** * Handle PUT /{bucket} for bucket creation. */ @@@ -83,12 -57,6 +83,12 @@@ Response handlePutRequest(S3RequestContext context, String bucketName, InputStream body) throws IOException, OS3Exception { + final String lifecycleMarker = queryParams().get(QueryParams.LIFECYCLE); + if (lifecycleMarker != null) { + context.setAction(S3GAction.PUT_BUCKET_LIFECYCLE); - return putBucketLifecycleConfiguration(bucketName, body); ++ return putBucketLifecycleConfiguration(context, bucketName, body); + } + if (!shouldHandle()) { return null; } @@@ -114,12 -82,6 +114,12 @@@ Response handleDeleteRequest(S3RequestContext context, String bucketName) throws IOException, OS3Exception { + final String lifecycleMarker = queryParams().get(QueryParams.LIFECYCLE); + if (lifecycleMarker != null) { + context.setAction(S3GAction.DELETE_BUCKET_LIFECYCLE); - return deleteBucketLifecycleConfiguration(bucketName); ++ return deleteBucketLifecycleConfiguration(context, bucketName); + } + if (!shouldHandle()) { return null; } @@@ -142,91 -104,4 +142,93 @@@ .status(HttpStatus.SC_NO_CONTENT) .build(); } + - public Response deleteBucketLifecycleConfiguration(String bucketName) throws IOException, OS3Exception { - verifyBucketOwner(bucketName); - deleteLifecycleConfiguration(bucketName); ++ public Response deleteBucketLifecycleConfiguration(S3RequestContext context, String bucketName) ++ throws IOException, OS3Exception { ++ verifyBucketOwner(context, bucketName); ++ deleteLifecycleConfiguration(context, bucketName); + return Response.noContent().build(); + } + - protected void deleteLifecycleConfiguration(String bucketName) ++ protected void deleteLifecycleConfiguration(S3RequestContext context, String bucketName) + throws IOException, OS3Exception { + try { - getBucket(bucketName).deleteLifecycleConfiguration(); ++ context.getVolume().getBucket(bucketName).deleteLifecycleConfiguration(); + } catch (OMException ex) { + if (ex.getResult() == OMException.ResultCodes.LIFECYCLE_CONFIGURATION_NOT_FOUND) { + throw S3ErrorTable.newError( + S3ErrorTable.NO_SUCH_LIFECYCLE_CONFIGURATION, bucketName); + } + throw ex; + } + } + - private void verifyBucketOwner(String bucketName) throws OS3Exception { ++ private void verifyBucketOwner(S3RequestContext context, String bucketName) throws OS3Exception { + HttpHeaders httpHeaders = getHeaders(); + if (httpHeaders == null) { + return; + } + String expectedBucketOwner = httpHeaders.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER); + if (expectedBucketOwner == null || expectedBucketOwner.isEmpty()) { + return; + } + + try { - String actualOwner = getBucket(bucketName).getOwner(); ++ String actualOwner = context.getVolume().getBucket(bucketName).getOwner(); + if (actualOwner != null && !actualOwner.equals(expectedBucketOwner)) { + LOG.debug("Bucket: {}, ExpectedBucketOwner: {}, ActualBucketOwner: {}", + bucketName, expectedBucketOwner, actualOwner); + throw S3ErrorTable.newError(S3ErrorTable.ACCESS_DENIED, bucketName); + } + } catch (Exception ex) { + LOG.error("Owner verification failed for bucket: {}", bucketName, ex); + throw S3ErrorTable.newError(S3ErrorTable.ACCESS_DENIED, bucketName); + } + } + - public Response putBucketLifecycleConfiguration(String bucketName, InputStream body) ++ public Response putBucketLifecycleConfiguration(S3RequestContext context, String bucketName, InputStream body) + throws IOException, OS3Exception { - verifyBucketOwner(bucketName); ++ verifyBucketOwner(context, bucketName); + S3LifecycleConfiguration s3LifecycleConfiguration; - OzoneBucket ozoneBucket = getBucket(bucketName); ++ OzoneBucket ozoneBucket = context.getVolume().getBucket(bucketName); + try { + s3LifecycleConfiguration = new PutBucketLifecycleConfigurationUnmarshaller().readFrom(null, + null, null, null, null, body); + OmLifecycleConfiguration lcc = + s3LifecycleConfiguration.toOmLifecycleConfiguration(ozoneBucket); + ozoneBucket.setLifecycleConfiguration(lcc); + } catch (WebApplicationException ex) { + throw S3ErrorTable.newError(S3ErrorTable.MALFORMED_XML, bucketName); + } catch (OMException ex) { + if (ex.getResult() == OMException.ResultCodes.ACCESS_DENIED) { + throw S3ErrorTable.newError(S3ErrorTable.ACCESS_DENIED, bucketName); + } else if (ex.getResult() == OMException.ResultCodes.INVALID_REQUEST) { + throw S3ErrorTable.newError(S3ErrorTable.INVALID_REQUEST, bucketName, ex); + } + } + return Response.ok().build(); + } + - public Response getBucketLifecycleConfiguration(String bucketName) throws IOException, OS3Exception { - verifyBucketOwner(bucketName); ++ public Response getBucketLifecycleConfiguration(S3RequestContext context, String bucketName) ++ throws IOException, OS3Exception { ++ verifyBucketOwner(context, bucketName); + OzoneLifecycleConfiguration ozoneLifecycleConfiguration = - getLifecycleConfiguration(bucketName); ++ getLifecycleConfiguration(context, bucketName); + return Response.ok(S3LifecycleConfiguration.fromOzoneLifecycleConfiguration( + ozoneLifecycleConfiguration), MediaType.APPLICATION_XML_TYPE).build(); + } + + protected OzoneLifecycleConfiguration getLifecycleConfiguration( - String bucketName) throws IOException, OS3Exception { ++ S3RequestContext context, String bucketName) throws IOException, OS3Exception { + try { - OzoneBucket ozoneBucket = getBucket(bucketName); ++ OzoneBucket ozoneBucket = context.getVolume().getBucket(bucketName); + return ozoneBucket.getLifecycleConfiguration(); + } catch (OMException ex) { + if (ex.getResult() == OMException.ResultCodes.LIFECYCLE_CONFIGURATION_NOT_FOUND) { + throw S3ErrorTable.newError( + S3ErrorTable.NO_SUCH_LIFECYCLE_CONFIGURATION, bucketName); + } + throw ex; + } + } } diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java index fbfb72afed0,8a8864c398b..dc27ca6ad53 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java @@@ -18,9 -18,9 +18,10 @@@ package org.apache.hadoop.ozone.client; import static org.apache.hadoop.ozone.OzoneConsts.ETAG; + import static org.apache.hadoop.ozone.OzoneConsts.EXPECTED_GEN_CREATE_IF_ABSENT; import static org.apache.hadoop.ozone.OzoneConsts.MD5_HASH; import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER; +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.LIFECYCLE_CONFIGURATION_NOT_FOUND; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
