This is an automated email from the ASF dual-hosted git repository. errose28 pushed a commit to branch HDDS-14496-zdu in repository https://gitbox.apache.org/repos/asf/ozone.git
commit a9a64dbed31ba1dfee9057b7c8ddd42acb948b2f Merge: c064bafb5e1 a99fabb9dea Author: Ethan Rose <[email protected]> AuthorDate: Tue Oct 6 13:56:01 2026 -0400 Merge branch 'master' into HDDS-14496-zdu * master: (64 commits) HDDS-16716. Add description for ozone.scm.ec.pipeline.per.volume.factor (#11415) HDDS-16008. PutBlocks from Flushes also go without Raft (#11356) HDDS-16666. Flush SCM transaction in memory during apply transaction (#11409) HDDS-15749. Run specific JUnit tests if possible (#10671) HDDS-16362. GetObjectAttributes ObjectParts should return Part entries for FSO buckets (#11242). HDDS-16643. Remove CleanupTableInfo mechanism (#11365) HDDS-16721. StreamBlockInputStream.read() returns a negative value for bytes 0x80 to 0xFF (#11411) HDDS-16241. gRPC deadline kills long-lived block streams after 30 seconds and the client never recovers (#11080) HDDS-15991. Speed up deleted table scans in quota repair (#11386) HDDS-16674. Bump awssdk to 2.55.6 (#11407) HDDS-16673. Avoid redundant ListBuckets RPCs when S3 bucket listing reaches the end (#11397) HDDS-16708. Let dependabot ignore iceberg minor version upgrades (#11398) HDDS-16713. Bump develocity-maven-extension to 2.6.0 (#11405) HDDS-16300. Allow OM to dynamically reconfigure its SCM node list without a restart (#11218) HDDS-15089. Support S3 per request read consistency (#11252) HDDS-16704. ReadBlock fails with IllegalStateException when a response is shorter than responseDataSize (#11402) HDDS-16631. Fix chooseRandom for rack names with common prefixes (#11401) HDDS-16654. Replace usage of deprecated finalize() in OM (#11376) HDDS-16658. Reuse source key details when opening input stream in S3 CopyObject (#11396) HDDS-16711. Bump moment to 2.31.0 (#11373) ... Conflicts: hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/HealthyReadOnlyNodeHandler.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCancelPrepareResponse.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMPrepareResponse.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java .github/dependabot.yml | 8 +- .github/workflows/build-ratis.yml | 2 +- .github/workflows/check.yml | 2 +- .github/workflows/ci.yml | 35 +- .github/workflows/intermittent-test-check.yml | 2 +- .github/workflows/label-pr.yml | 2 +- .github/workflows/populate-cache.yml | 2 +- .github/workflows/repeat-acceptance.yml | 2 +- .mvn/extensions.xml | 2 +- dev-support/ci/selective_ci_checks.bats | 42 +- dev-support/ci/selective_ci_checks.sh | 68 +++- .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 23 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 5 +- .../hdds/scm/storage/BlockDataStreamOutput.java | 77 +++- .../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +- .../hdds/scm/storage/MultipartInputStream.java | 48 +-- .../hadoop/hdds/scm/storage/PartInputStream.java | 13 + .../hdds/scm/storage/StreamBlockInputStream.java | 182 +++++++-- .../hadoop/hdds/scm/TestOzoneClientConfig.java | 10 +- .../hdds/scm/storage/MockDatanodePipeline.java | 45 +- .../scm/storage/TestBlockDataStreamOutput.java | 59 ++- .../hdds/scm/storage/TestMultipartInputStream.java | 69 ++++ .../scm/storage/TestStreamBlockInputStream.java | 361 ++++++++++++++++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 + .../hadoop/hdds/utils/ConnectionFailureUtils.java | 2 +- .../common/src/main/resources/ozone-default.xml | 23 +- .../hdds/utils/TestConnectionFailureUtils.java | 14 + .../common/statemachine/DatanodeConfiguration.java | 22 + .../ClosePipelineCommandHandler.java | 4 +- .../transport/server/GrpcXceiverService.java | 68 +++- .../common/transport/server/XceiverServerGrpc.java | 5 +- .../server/ratis/ContainerStateMachine.java | 31 +- .../transport/server/ratis/DispatcherContext.java | 4 +- .../common/transport/server/ratis/LocalStream.java | 22 +- .../container/keyvalue/ReadBlockComputation.java | 8 +- .../container/common/impl/TestHddsDispatcher.java | 3 +- .../transport/server/TestGrpcXceiverService.java | 169 ++++++++ .../container/keyvalue/TestKeyValueHandler.java | 18 + .../docs/content/feature/Reconfigurability.md | 2 + hadoop-hdds/framework/pom.xml | 22 +- .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 2 +- .../scm/proxy/SCMFailoverProxyProviderBase.java | 91 ++++- .../hdds/server/http/AdminAuthorizedServlet.java | 2 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 56 ++- ...horizedServlet.java => HddsJMXJsonServlet.java} | 28 +- .../hadoop/hdds/server/http/HttpServer2.java | 244 ++++++++--- .../hdds/server/http/ServletElementsFactory.java | 4 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 27 +- .../apache/hadoop/hdds/utils/db/TypedTable.java | 2 + .../utils/db/cache/TableCacheUpdateTracker.java | 88 ++++ .../hdds/scm/net/TestNetworkTopologyImpl.java | 19 + .../TestSCMFailoverProxyProviderChangeConfig.java | 149 +++++++ .../hdds/server/http/TestBaseHttpServer.java | 53 +++ .../hadoop/hdds/server/http/TestHttpServer2.java | 390 ++++++++++++++++++ .../hdds/server/http/TestHttpServer2SSL.java | 36 +- .../hdds/utils/db/TestTableCacheUpdateTracker.java | 118 ++++++ hadoop-hdds/rocks-native/pom.xml | 2 +- hadoop-hdds/server-scm/pom.xml | 12 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +- .../hdds/scm/ha/SCMHADBTransactionBuffer.java | 2 + .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 41 ++ .../hdds/scm/ha/SCMHADBTransactionBufferStub.java | 5 + .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 2 + .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 17 +- .../hadoop/hdds/scm/node/DeadNodeHandler.java | 24 -- .../hadoop/hdds/scm/node/NodeStateManager.java | 32 +- .../hdds/scm/node/PendingContainerTracker.java | 2 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 221 ++++++---- .../scm/node/UnhealthyToHealthyNodeHandler.java | 15 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 2 +- .../scm/pipeline/WritableECContainerProvider.java | 5 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +- .../hdds/scm/server/StorageContainerManager.java | 2 +- .../scm/ha/TestSCMHADBTransactionBufferImpl.java | 333 +++++++++++++++ .../ha/TestSCMHATransactionBufferMonitorTask.java | 1 + .../hadoop/hdds/scm/ha/TestSCMStateMachine.java | 124 +++++- .../hdds/scm/ha/TestSequenceIDGenerator.java | 53 +++ .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 57 +-- .../hadoop/hdds/scm/node/TestNodeStateManager.java | 88 +++- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 63 ++- .../admin/om/BucketDeletedBytesSubCommand.java | 101 +++++ .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 + .../ozone/client/S3HeadObjectAttributes.java | 10 +- .../ozone/client/io/OzoneCryptoInputStream.java | 5 + .../ozone/client/protocol/ClientProtocol.java | 28 ++ .../apache/hadoop/ozone/client/rpc/RpcClient.java | 41 +- .../hadoop/ozone/client/rpc/TestRpcClient.java | 17 + .../main/java/org/apache/hadoop/ozone/OmUtils.java | 3 + ...doopRpcOMFollowerReadFailoverProxyProvider.java | 73 +++- .../ozone/om/ha/OMFailoverProxyProviderBase.java | 21 + .../ozone/om/helpers/BucketDeletedBytes.java | 79 ++++ .../ozone/om/helpers/KeyInfoWithVolumeContext.java | 25 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 8 + .../ozone/om/protocolPB/GrpcOmTransport.java | 57 ++- .../ozone/om/protocolPB/Hadoop3OmTransport.java | 4 +- .../om/protocolPB/OzoneManagerClientProtocol.java | 10 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 86 +++- .../java/org/apache/hadoop/ozone/TestOmUtils.java | 10 + ...doopRpcOMFollowerReadFailoverProxyProvider.java | 114 +++++- ...OzoneManagerProtocolClientSideTranslatorPB.java | 108 +++++ .../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 119 ++++++ hadoop-ozone/dev-support/checks/integration.sh | 26 +- hadoop-ozone/dev-support/checks/junit.sh | 3 +- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 16 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 16 +- .../src/main/smoketest/s3/secretgenerate.robot | 3 +- hadoop-ozone/httpfsgateway/pom.xml | 12 +- .../fs/http/server/HttpFSServerWebServer.java | 4 + .../main/resources/webapps/webhdfs/WEB-INF/web.xml | 10 +- .../httpfsgateway/src/main/webapp/WEB-INF/web.xml | 10 +- hadoop-ozone/iceberg/pom.xml | 24 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 28 +- hadoop-ozone/integration-test-s3/pom.xml | 40 +- .../hadoop/ozone/s3/MultiS3GatewayService.java | 21 +- .../org/apache/hadoop/ozone/s3/ProxyServer.java | 22 +- .../ozone/s3/ProxyServerIntegrationTest.java | 4 +- .../apache/hadoop/ozone/s3/S3ClientFactory.java | 1 + .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 115 +++++- .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 453 ++++++++++++++++++++- hadoop-ozone/integration-test/pom.xml | 50 +-- .../hdds/scm/TestOmSCMNodesReconfiguration.java | 363 +++++++++++++++++ .../apache/hadoop/hdds/scm/node/TestQueryNode.java | 23 ++ .../hadoop/hdds/scm/storage/TestCommitWatcher.java | 19 +- .../org/apache/hadoop/ozone/TestCpuMetrics.java | 5 +- .../client/rpc/TestBlockDataStreamOutput.java | 42 +- .../rpc/TestContainerStateMachineStream.java | 10 +- .../client/rpc/TestOzoneAtRestEncryption.java | 127 ++++-- .../rpc/read/TestStreamBlockInputStream.java | 88 ++-- .../ozone/om/OzoneManagerHAFollowerReadTests.java | 8 +- .../ozone/om/TestGetDBUpdatesAuthorization.java | 73 +++- ...stOzoneManagerHAFollowerReadWithAllRunning.java | 45 +- ...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +- .../ozone/om/TestOzoneManagerHAWithAllRunning.java | 4 +- .../ozone/om/TestOzoneManagerRestInterface.java | 8 +- .../TestOmMixedWorkloadUnderDeletionBench.java | 19 +- ...TestSnapshotDeletingServiceIntegrationTest.java | 125 +++--- .../hadoop/ozone/shell/TestOzoneShellHA.java | 56 ++- .../src/main/proto/OmClientProtocol.proto | 35 +- hadoop-ozone/ozone-manager/pom.xml | 12 +- .../apache/hadoop/ozone/om/OmMetadataReader.java | 6 +- .../org/apache/hadoop/ozone/om/OmSnapshot.java | 35 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 226 ++++++++-- .../apache/hadoop/ozone/om/S3SecretManager.java | 4 + .../java/org/apache/hadoop/ozone/om/ScmClient.java | 37 ++ .../hadoop/ozone/om/execution/OMExecutionFlow.java | 20 +- .../hadoop/ozone/om/lock/OMLockDetailsUtil.java | 48 +++ .../hadoop/ozone/om/lock/OzoneManagerLock.java | 7 +- .../hadoop/ozone/om/ratis/OMThreadContext.java | 198 +++++++++ .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 26 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 53 +-- .../hadoop/ozone/om/request/OMClientRequest.java | 26 +- .../file/OMDirectoryCreateRequestWithFSO.java | 6 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 30 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +- .../hadoop/ozone/om/response/CleanupTableInfo.java | 51 --- .../ozone/om/response/DummyOMClientResponse.java | 1 - .../hadoop/ozone/om/response/OMClientResponse.java | 24 +- .../om/response/bucket/OMBucketCreateResponse.java | 5 - .../om/response/bucket/OMBucketDeleteResponse.java | 6 - .../response/bucket/OMBucketSetOwnerResponse.java | 4 - .../bucket/OMBucketSetPropertyResponse.java | 4 - .../response/bucket/acl/OMBucketAclResponse.java | 4 - .../response/file/OMDirectoryCreateResponse.java | 4 - .../file/OMDirectoryCreateResponseWithFSO.java | 14 +- .../om/response/file/OMFileCreateResponse.java | 5 - .../response/file/OMFileCreateResponseWithFSO.java | 7 - .../om/response/file/OMRecoverLeaseResponse.java | 5 - .../om/response/key/OMAllocateBlockResponse.java | 5 - .../key/OMAllocateBlockResponseWithFSO.java | 5 - .../key/OMDirectoriesPurgeResponseWithFSO.java | 8 - .../ozone/om/response/key/OMKeyCommitResponse.java | 8 - .../response/key/OMKeyCommitResponseWithFSO.java | 8 - .../ozone/om/response/key/OMKeyCreateResponse.java | 6 - .../response/key/OMKeyCreateResponseWithFSO.java | 7 - .../ozone/om/response/key/OMKeyDeleteResponse.java | 7 - .../response/key/OMKeyDeleteResponseWithFSO.java | 10 - .../ozone/om/response/key/OMKeyPurgeResponse.java | 4 - .../ozone/om/response/key/OMKeyRenameResponse.java | 5 - .../response/key/OMKeyRenameResponseWithFSO.java | 7 - .../om/response/key/OMKeySetTimesResponse.java | 4 - .../response/key/OMKeySetTimesResponseWithFSO.java | 5 - .../om/response/key/OMKeysDeleteResponse.java | 7 - .../response/key/OMKeysDeleteResponseWithFSO.java | 11 - .../om/response/key/OMKeysRenameResponse.java | 4 - .../om/response/key/OMOpenKeysDeleteResponse.java | 8 - .../om/response/key/acl/OMKeyAclResponse.java | 4 - .../response/key/acl/OMKeyAclResponseWithFSO.java | 5 - .../key/acl/prefix/OMPrefixAclResponse.java | 4 - .../OMLifecycleConfigurationDeleteResponse.java | 4 - .../OMLifecycleConfigurationSetResponse.java | 4 - .../OMLifecycleSaveScanStateResponse.java | 3 - .../OMLifecycleSetServiceStatusResponse.java | 2 - .../S3ExpiredMultipartUploadsAbortResponse.java | 10 - .../S3InitiateMultipartUploadResponse.java | 5 - .../S3InitiateMultipartUploadResponseWithFSO.java | 7 - .../multipart/S3MultipartUploadAbortResponse.java | 9 - .../S3MultipartUploadAbortResponseWithFSO.java | 9 - .../S3MultipartUploadCommitPartResponse.java | 8 - ...S3MultipartUploadCommitPartResponseWithFSO.java | 9 - .../S3MultipartUploadCompleteResponse.java | 10 - .../S3MultipartUploadCompleteResponseWithFSO.java | 10 - .../response/s3/security/OMSetSecretResponse.java | 4 - .../response/s3/security/S3AssumeRoleResponse.java | 2 - .../security/S3DeleteRevokedSTSTokensResponse.java | 3 - .../response/s3/security/S3GetSecretResponse.java | 4 - .../s3/security/S3RevokeSTSTokenResponse.java | 3 - .../s3/security/S3RevokeSecretResponse.java | 4 - .../s3/tagging/S3DeleteObjectTaggingResponse.java | 4 - .../S3DeleteObjectTaggingResponseWithFSO.java | 4 - .../s3/tagging/S3PutObjectTaggingResponse.java | 4 - .../tagging/S3PutObjectTaggingResponseWithFSO.java | 4 - .../tenant/OMSetRangerServiceVersionResponse.java | 4 - .../s3/tenant/OMTenantAssignAdminResponse.java | 6 - .../tenant/OMTenantAssignUserAccessIdResponse.java | 10 - .../response/s3/tenant/OMTenantCreateResponse.java | 8 - .../response/s3/tenant/OMTenantDeleteResponse.java | 8 - .../s3/tenant/OMTenantRevokeAdminResponse.java | 6 - .../tenant/OMTenantRevokeUserAccessIdResponse.java | 10 - .../security/OMCancelDelegationTokenResponse.java | 4 - .../security/OMGetDelegationTokenResponse.java | 4 - .../security/OMRenewDelegationTokenResponse.java | 4 - .../snapshot/OMSnapshotCreateResponse.java | 7 - .../snapshot/OMSnapshotDeleteResponse.java | 4 - .../OMSnapshotMoveDeletedKeysResponse.java | 3 - .../snapshot/OMSnapshotMoveTableKeysResponse.java | 3 - .../response/snapshot/OMSnapshotPurgeResponse.java | 8 - .../snapshot/OMSnapshotRenameResponse.java | 4 - .../snapshot/OMSnapshotSetPropertyResponse.java | 4 - .../upgrade/OMCompleteFinalizeUpgradeResponse.java | 3 - .../upgrade/OMStartFinalizeUpgradeResponse.java | 4 - .../om/response/util/OMEchoRPCWriteResponse.java | 2 - .../om/response/volume/OMQuotaRepairResponse.java | 5 - .../om/response/volume/OMVolumeAclOpResponse.java | 4 - .../om/response/volume/OMVolumeCreateResponse.java | 4 - .../om/response/volume/OMVolumeDeleteResponse.java | 4 - .../response/volume/OMVolumeSetOwnerResponse.java | 4 - .../response/volume/OMVolumeSetQuotaResponse.java | 4 - .../ozone/om/service/KeyLifecycleService.java | 10 +- .../hadoop/ozone/om/service/QuotaRepairTask.java | 129 +++++- .../ozone/om/snapshot/SnapshotDiffManager.java | 15 +- .../trapped/BucketDeletedDataCalculator.java | 373 +++++++++++++++++ .../ozone/om/snapshot/trapped/package-info.java | 19 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 46 ++- .../protocolPB/OzoneManagerRequestHandler.java | 27 +- .../hadoop/ozone/om/TestEDEKCacheLoader.java | 78 ++++ .../ozone/om/TestOmSnapshotLeakDetection.java | 87 ++++ .../org/apache/hadoop/ozone/om/TestScmClient.java | 24 ++ .../ozone/om/execution/TestOMExecutionFlow.java | 165 ++++++++ .../ozone/om/lock/TestOMLockDetailsUtil.java | 78 ++++ .../hadoop/ozone/om/ratis/TestOMThreadContext.java | 207 ++++++++++ ...tOzoneManagerDoubleBufferWithDummyResponse.java | 74 +++- .../om/ratis/TestOzoneManagerStateMachine.java | 226 ++++++++++ .../om/request/key/TestOMKeyCreateRequest.java | 9 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 7 + .../ozone/om/response/TestCleanupTableInfo.java | 384 ----------------- .../ozone/om/response/TestOMClientResponse.java | 51 +++ .../hadoop/ozone/om/response/TestOMResponse.java | 51 ++- .../hadoop/ozone/om/response/TestS3DerivedKey.java | 241 +++++++++++ .../file/TestOMDirectoryCreateResponseWithFSO.java | 2 - .../ozone/om/service/TestKeyLifecycleService.java | 10 +- .../ozone/om/service/TestQuotaRepairTask.java | 425 +++++++++++++++++++ .../ozone/om/snapshot/TestSnapshotDiffManager.java | 18 + .../trapped/TestBucketDeletedDataCalculator.java | 243 +++++++++++ hadoop-ozone/recon/pom.xml | 12 +- .../ozone/recon/api/ClusterStateEndpoint.java | 2 +- .../ozone/recon/api/filters/ReconAuthFilter.java | 2 +- .../webapps/recon/ozone-recon-web/package.json | 4 +- .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 34 +- .../src/v2/components/navBar/navBar.tsx | 12 +- .../src/v2/pages/heatmap/heatmap.tsx | 35 +- hadoop-ozone/s3gateway/pom.xml | 26 +- .../org/apache/hadoop/ozone/audit/S3GAction.java | 6 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 3 +- .../hadoop/ozone/s3/ReadConsistencyFilter.java | 53 +++ .../hadoop/ozone/s3/S3ContentTypeFilter.java | 135 ++++++ .../hadoop/ozone/s3/S3GatewayHttpServer.java | 13 + .../hadoop/ozone/s3/S3GatewayWebAdminServer.java | 9 +- .../apache/hadoop/ozone/s3/S3STSHttpServer.java | 3 + .../hadoop/ozone/s3/SignedChunksInputStream.java | 152 ++++++- .../apache/hadoop/ozone/s3/WeldCdiIntegration.java | 50 +++ .../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 5 + .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 6 + .../s3/endpoint/BucketGetLocationHandler.java | 5 + .../ozone/s3/endpoint/BucketLifecycleHandler.java | 9 +- .../ozone/s3/endpoint/BucketOperationHandler.java | 21 +- .../BucketOperationNotImplementedHandler.java | 64 +++ .../hadoop/ozone/s3/endpoint/EndpointBase.java | 59 ++- .../s3/endpoint/ListMultipartUploadsHandler.java | 15 +- .../ozone/s3/endpoint/MultipartKeyHandler.java | 12 +- .../hadoop/ozone/s3/endpoint/ObjectAclHandler.java | 15 +- .../ozone/s3/endpoint/ObjectAttributesHandler.java | 43 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 16 +- .../ozone/s3/endpoint/ObjectGetTorrentHandler.java | 5 + .../ozone/s3/endpoint/ObjectOperationHandler.java | 17 +- .../ObjectOperationNotImplementedHandler.java | 65 +++ .../ozone/s3/endpoint/ObjectTaggingHandler.java | 19 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 8 +- .../hadoop/ozone/s3/endpoint/S3RequestContext.java | 12 + .../hadoop/ozone/s3/exception/S3ErrorTable.java | 9 + .../ozone/s3/signature/AWSSignatureProcessor.java | 3 +- .../s3/signature/AuthorizationV4HeaderParser.java | 43 +- .../s3/signature/AuthorizationV4QueryParser.java | 7 +- .../hadoop/ozone/s3/signature/ChunksValidator.java | 39 +- .../hadoop/ozone/s3/signature/Credential.java | 20 + .../s3/signature/MalformedResourceException.java | 19 +- .../ozone/s3/signature/StringToSignProducer.java | 50 --- .../ozone/s3/util/ReadConsistencyContext.java | 112 +++++ .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 45 ++ .../hadoop/ozone/s3/util/S3GActionIamMapper.java | 2 + .../ozone/s3secret/S3SecretManagementEndpoint.java | 17 +- .../resources/webapps/s3gateway/WEB-INF/web.xml | 8 + .../hadoop/ozone/client/ClientProtocolStub.java | 42 +- .../hadoop/ozone/client/OzoneBucketStub.java | 2 +- .../hadoop/ozone/s3/TestAuthorizationFilter.java | 94 +++-- .../hadoop/ozone/s3/TestOzoneClientProducer.java | 58 +++ .../hadoop/ozone/s3/TestReadConsistencyFilter.java | 51 +++ .../hadoop/ozone/s3/TestS3ContentTypeFilter.java | 127 ++++++ .../hadoop/ozone/s3/TestS3GatewayHttpServers.java | 100 +++++ .../ozone/s3/TestSignedChunksInputStream.java | 173 ++++++++ .../ozone/s3/endpoint/EndpointTestUtils.java | 20 + .../hadoop/ozone/s3/endpoint/TestBucketList.java | 389 ++++++++---------- .../s3/endpoint/TestBucketNotImplemented.java | 102 +++++ .../hadoop/ozone/s3/endpoint/TestEndpointBase.java | 189 +++++++++ .../s3/endpoint/TestListMultipartUploads.java | 166 ++++++++ .../hadoop/ozone/s3/endpoint/TestListParts.java | 108 +++++ .../ozone/s3/endpoint/TestObjectAttributesGet.java | 146 ++++++- .../ozone/s3/endpoint/TestObjectMultiDelete.java | 24 ++ .../s3/endpoint/TestObjectNotImplemented.java | 113 +++++ .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 130 +++++- .../ozone/s3/endpoint/TestObjectTaggingDelete.java | 39 +- .../ozone/s3/endpoint/TestObjectTaggingGet.java | 20 + .../ozone/s3/endpoint/TestObjectTaggingPut.java | 35 +- .../hadoop/ozone/s3/endpoint/TestPartUpload.java | 39 ++ .../ozone/s3/endpoint/TestPermissionCheck.java | 10 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 111 ++++- .../TestS3LifecycleConfigurationDelete.java | 125 ++++++ .../endpoint/TestS3LifecycleConfigurationGet.java | 90 ++++ .../endpoint/TestS3LifecycleConfigurationPut.java | 90 ++++ .../ozone/s3/endpoint/TestS3RequestContext.java | 86 ++++ .../ozone/s3/endpoint/TestUploadWithStream.java | 34 ++ .../ozone/s3/signature/SignatureTestUtils.java | 34 ++ .../signature/TestAuthorizationV4HeaderParser.java | 133 +++--- .../signature/TestAuthorizationV4QueryParser.java | 55 ++- .../ozone/s3/signature/TestChunksValidator.java | 56 +++ .../s3/signature/TestStringToSignProducer.java | 56 +-- .../ozone/s3/util/TestS3GActionIamMapper.java | 2 + .../hadoop/ozone/s3secret/TestSecretGenerate.java | 6 +- .../hadoop/ozone/s3secret/TestSecretRevoke.java | 5 + pom.xml | 77 ++-- 350 files changed, 12802 insertions(+), 2653 deletions(-) diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java index cd5d01d05a3,51df2cf7879..0563f9d6eea --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java @@@ -148,10 -166,15 +153,12 @@@ public class NodeStateManager implement * * @param conf Configuration * @param eventPublisher event publisher + * @param clusterMap network topology of the cluster - * @param layoutManager Layout version manager */ public NodeStateManager(ConfigurationSource conf, EventPublisher eventPublisher, + NetworkTopology clusterMap, - LayoutVersionManager layoutManager, SCMContext scmContext) { - this.layoutVersionManager = layoutManager; this.nodeStateMap = new NodeStateMap(); this.node2PipelineMap = new Node2PipelineMap(); this.eventPublisher = eventPublisher; @@@ -237,13 -314,13 +245,13 @@@ * @throws NodeAlreadyExistsException if the node is already present */ public void addNode(DatanodeDetails datanodeDetails, - LayoutVersionProto layoutInfo) throws NodeAlreadyExistsException { - nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, layoutInfo)); + DatanodeVersionProto versionInfo) throws NodeAlreadyExistsException { + nodeStateMap.addNode(newDatanodeInfo(datanodeDetails, versionInfo)); try { - updateLastKnownLayoutVersion(datanodeDetails, layoutInfo); + updateLastKnownVersionInfo(datanodeDetails, versionInfo); } catch (NodeNotFoundException ex) { throw new IllegalStateException("Inconsistent NodeStateMap! Datanode " - + datanodeDetails.getID() + " was added but not found in map: " + nodeStateMap); + + datanodeDetails + " was added but not found in map: " + nodeStateMap); } } diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java index cd6a0e1fdf1,d8884cdc464..3b640d6e8cc --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java @@@ -193,10 -192,11 +193,10 @@@ public class SCMNodeManager implements EventPublisher eventPublisher, NetworkTopology networkTopology, SCMContext scmContext, - HDDSLayoutVersionManager layoutVersionManager, + ScmVersionManager versionManager, Function<String, String> nodeResolver) { this.scmNodeEventPublisher = eventPublisher; - this.nodeStateManager = new NodeStateManager(conf, eventPublisher, scmContext); - this.nodeStateManager = new NodeStateManager(conf, eventPublisher, networkTopology, - layoutVersionManager, scmContext); ++ this.nodeStateManager = new NodeStateManager(conf, eventPublisher, networkTopology, scmContext); this.version = VersionInfo.getLatestVersion(); this.commandQueue = new CommandQueue(); this.scmStorageConfig = scmStorageConfig; @@@ -473,10 -476,7 +473,7 @@@ if (updateDnsToDnIdMap(oldNode.getHostName(), oldNode.getIpAddress(), hostName, ipAddress, dnId)) { LOG.info("Updating datanode from {} to {}", oldNode, datanodeDetails); - clusterMap.update(oldNode, datanodeDetails); - nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); - DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); - Preconditions.checkState(dn.getParent() != null); - DatanodeDetails dn = updateNodeAndTopology(oldNode, datanodeDetails, layoutInfo); ++ DatanodeDetails dn = updateNodeAndTopology(oldNode, datanodeDetails, dnVersionInfo); processNodeReport(datanodeDetails, nodeReport); LOG.info("Updated datanode to: {}", dn); scmNodeEventPublisher.fireEvent(SCMEvents.NODE_ADDRESS_UPDATE, dn); @@@ -484,12 -484,12 +481,12 @@@ LOG.info("Update the version for registered datanode {}, " + "oldVersion = {}, newVersion = {}.", datanodeDetails, oldNode.getVersion(), datanodeDetails.getVersion()); - nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); - updateNodeAndTopology(oldNode, datanodeDetails, layoutInfo); ++ updateNodeAndTopology(oldNode, datanodeDetails, dnVersionInfo); } else if (oldNode.portsChanged(datanodeDetails)) { // Refresh the stored node when its port set changes LOG.info("Updating ports for registered datanode {}: {} -> {}", datanodeDetails, oldNode.getPorts(), datanodeDetails.getPorts()); - nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); - updateNodeAndTopology(oldNode, datanodeDetails, layoutInfo); ++ updateNodeAndTopology(oldNode, datanodeDetails, dnVersionInfo); } } catch (NodeNotFoundException e) { LOG.error("Cannot find datanode {} from nodeStateManager", @@@ -503,6 -503,25 +500,25 @@@ .build(); } + /** + * Replaces the record of an already registered datanode, in both the node state and the network + * topology. The topology has to be updated first: {@link NetworkTopology#update} sets the parent on + * the new node, and {@link NodeStateManager#updateNode} only picks it up because the stored + * {@link DatanodeInfo} is copied from that node afterwards. Updating the node state alone would + * leave the stale record in the topology and the stored node without a parent. + * + * @return the refreshed node as held by the node state manager + */ + private DatanodeDetails updateNodeAndTopology(DatanodeInfo oldNode, - DatanodeDetails datanodeDetails, LayoutVersionProto layoutInfo) ++ DatanodeDetails datanodeDetails, DatanodeVersionProto dnVersionInfo) + throws NodeNotFoundException { + clusterMap.update(oldNode, datanodeDetails); - nodeStateManager.updateNode(datanodeDetails, layoutInfo); ++ nodeStateManager.updateNode(datanodeDetails, dnVersionInfo); + final DatanodeDetails dn = nodeStateManager.getNode(datanodeDetails); + Preconditions.checkState(dn.getParent() != null); + return dn; + } + /** * Add an entry to the dnsToUuidMap, which maps hostname / IP to the DNs * running on that host. As each address can have many DNs running on it, diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/UnhealthyToHealthyNodeHandler.java index 56bda449847,eba89247a19..2d015ee723d --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/UnhealthyToHealthyNodeHandler.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/UnhealthyToHealthyNodeHandler.java @@@ -17,42 -17,35 +17,29 @@@ package org.apache.hadoop.hdds.scm.node; - import java.util.Objects; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.hdds.scm.ha.SCMService.Event; +import org.apache.hadoop.hdds.scm.ha.SCMService; import org.apache.hadoop.hdds.scm.ha.SCMServiceManager; - import org.apache.hadoop.hdds.scm.net.NetworkTopology; import org.apache.hadoop.hdds.server.events.EventHandler; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Handles Read Only healthy to healthy node event. (Possibly due to a - * datanode having finalized) + * Handles non healthy to healthy node event. */ -public class ReadOnlyHealthyToHealthyNodeHandler - implements EventHandler<DatanodeDetails> { - private static final Logger LOG = - LoggerFactory.getLogger(ReadOnlyHealthyToHealthyNodeHandler.class); +public class UnhealthyToHealthyNodeHandler implements EventHandler<DatanodeDetails> { + private static final Logger LOG = LoggerFactory.getLogger(UnhealthyToHealthyNodeHandler.class); private final SCMServiceManager serviceManager; - private final NodeManager nodeManager; - public UnhealthyToHealthyNodeHandler(NodeManager nodeManager, SCMServiceManager serviceManager) { - public ReadOnlyHealthyToHealthyNodeHandler(SCMServiceManager serviceManager) { ++ public UnhealthyToHealthyNodeHandler(SCMServiceManager serviceManager) { this.serviceManager = serviceManager; - this.nodeManager = nodeManager; } @Override - public void onMessage(DatanodeDetails datanodeDetails, - EventPublisher publisher) { - LOG.info("Datanode {} moved to HEALTHY state.", - datanodeDetails); - serviceManager.notifyEventTriggered( - Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED); + public void onMessage(DatanodeDetails datanodeDetails, EventPublisher publisher) { + LOG.info("Datanode {} moved to HEALTHY state.", datanodeDetails); - - //add node back if it is not present in networkTopology - NetworkTopology nt = nodeManager.getClusterNetworkTopologyMap(); - if (!nt.contains(datanodeDetails)) { - nt.add(datanodeDetails); - // make sure after DN is added back into topology, DatanodeDetails - // instance returned from nodeStateManager has parent correctly set. - Objects.requireNonNull(nodeManager.getNode(datanodeDetails.getID()).getParent(), "Parent == null"); - } + serviceManager.notifyEventTriggered(SCMService.Event.UNHEALTHY_TO_HEALTHY_NODE_HANDLER_TRIGGERED); } } diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 09e656b7a75,12665164c85..0fa0e38bfa7 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -507,8 -512,12 +507,8 @@@ public final class StorageContainerMana pipelineManager, containerManager, null); StartDatanodeAdminHandler datanodeStartAdminHandler = new StartDatanodeAdminHandler(scmNodeManager, pipelineManager); - ReadOnlyHealthyToHealthyNodeHandler readOnlyHealthyToHealthyNodeHandler = - new ReadOnlyHealthyToHealthyNodeHandler(serviceManager); - HealthyReadOnlyNodeHandler - healthyReadOnlyNodeHandler = - new HealthyReadOnlyNodeHandler(scmNodeManager, - pipelineManager); + UnhealthyToHealthyNodeHandler unhealthyToHealthyNodeHandler = - new UnhealthyToHealthyNodeHandler(scmNodeManager, serviceManager); ++ new UnhealthyToHealthyNodeHandler(serviceManager); ContainerActionsHandler actionsHandler = new ContainerActionsHandler(); ContainerReportHandler containerReportHandler = diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java index 314c676d7c0,0f6703cdf57..0bf82bad2b5 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMStateMachine.java @@@ -20,18 -20,26 +20,29 @@@ package org.apache.hadoop.hdds.scm.ha import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; + import static org.mockito.ArgumentMatchers.anyString; + import static org.mockito.Mockito.doThrow; + import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; + import static org.mockito.Mockito.times; + import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; + import java.util.concurrent.CompletableFuture; import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMMetrics; + import org.apache.hadoop.hdds.scm.exceptions.SCMException; + import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes; import org.apache.hadoop.hdds.scm.ha.invoker.ScmInvoker; + import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; + import org.apache.hadoop.hdds.scm.server.SCMDatanodeProtocolServer; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.utils.TransactionInfo; +import org.apache.hadoop.ozone.upgrade.UpgradeException; import org.apache.ratis.proto.RaftProtos; +import org.apache.ratis.proto.RaftProtos.LogEntryProto; +import org.apache.ratis.proto.RaftProtos.StateMachineLogEntryProto; + import org.apache.ratis.protocol.Message; import org.apache.ratis.server.protocol.TermIndex; import org.apache.ratis.statemachine.TransactionContext; import org.apache.ratis.util.ExitUtils; @@@ -59,48 -67,101 +71,128 @@@ public class TestSCMStateMachine metrics.unRegister(); } - + + /** + * A finalization step that throws an UpgradeException (an IOException, not an SCMException) must + * crash SCM rather than be returned to the Ratis client. UpgradeException skips the inner + * catch (SCMException) and hits the outer catch (Exception) -> ExitUtils.terminate. + */ + @Test + public void testUpgradeExceptionDuringApplyTerminates() throws Exception { + ExitUtils.disableSystemExit(); ++ try { ++ SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class); ++ ScmInvoker<?> invoker = mock(ScmInvoker.class); ++ when(invoker.invokeLocal(any(), any())).thenThrow( ++ new UpgradeException(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED)); ++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.FINALIZE, invoker); ++ ++ // terminate throws ExitException when system exit is disabled ++ assertThrows(ExitUtils.ExitException.class, ++ () -> stateMachine.applyTransaction(newTransaction(RequestType.FINALIZE, 1))); ++ } finally { ++ ExitUtils.clear(); ++ } ++ } + - StorageContainerManager scm = mock(StorageContainerManager.class); - SCMMetrics metrics = SCMMetrics.create(); - when(scm.getMetrics()).thenReturn(metrics); + @Test + public void testApplyTransactionFlushesAfterRecordingTransactionInfo() throws Exception { + SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class); - SCMStateMachine stateMachine = newStateMachine(buffer, succeedingInvoker()); ++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, succeedingInvoker()); + - CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(7)); ++ CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, 7)); + + assertTrue(result.isDone() && !result.isCompletedExceptionally()); + InOrder order = inOrder(buffer); + order.verify(buffer).beginApplyingTransaction(); + order.verify(buffer).updateLatestTrxInfo(TransactionInfo.valueOf(TermIndex.valueOf(1, 7))); + order.verify(buffer).flushIfPendingLimitReached(); + order.verify(buffer).endApplyingTransaction(); + } + @Test + public void testFlushCheckedForEveryAppliedTransaction() throws Exception { SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class); - when(buffer.getLatestTrxInfo()).thenReturn(TransactionInfo.valueOf(TermIndex.valueOf(0, 0))); - SCMStateMachine stateMachine = newStateMachine(buffer, succeedingInvoker()); ++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, succeedingInvoker()); - // The contents of the state machine are mocked, so SCMStateMachine#close is a no-op. - SCMStateMachine stateMachine = new SCMStateMachine(scm, buffer); + for (int i = 1; i <= 5; i++) { - stateMachine.applyTransaction(newTransaction(i)); ++ stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, i)); + } + + verify(buffer, times(5)).flushIfPendingLimitReached(); + verify(buffer, times(5)).endApplyingTransaction(); + } - ++ + @Test + public void testFlushStillCheckedWhenTransactionIsLogicallyRejected() throws Exception { + SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class); ScmInvoker<?> invoker = mock(ScmInvoker.class); - when(invoker.invokeLocal(any(), any())).thenThrow( - new UpgradeException(UpgradeException.ResultCodes.FINALIZE_UPGRADE_ACTION_FAILED)); - stateMachine.registerInvoker(RequestType.FINALIZE, invoker); + when(invoker.invokeLocal(anyString(), any())) + .thenThrow(new SCMException("rejected", ResultCodes.FAILED_TO_FIND_CONTAINER)); - SCMStateMachine stateMachine = newStateMachine(buffer, invoker); ++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, invoker); + - CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(3)); ++ CompletableFuture<Message> result = stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, 3)); + + assertTrue(result.isCompletedExceptionally()); + InOrder order = inOrder(buffer); + order.verify(buffer).updateLatestTrxInfo(TransactionInfo.valueOf(TermIndex.valueOf(1, 3))); + order.verify(buffer).flushIfPendingLimitReached(); + order.verify(buffer).endApplyingTransaction(); + } + + @Test + public void testFlushFailureTerminatesAndClosesApplyingWindow() throws Exception { + ExitUtils.disableSystemExit(); + try { + SCMHADBTransactionBuffer buffer = mock(SCMHADBTransactionBuffer.class); + doThrow(new IllegalStateException("injected flush failure")).when(buffer).flushIfPendingLimitReached(); - SCMStateMachine stateMachine = newStateMachine(buffer, succeedingInvoker()); ++ SCMStateMachine stateMachine = newStateMachine(buffer, RequestType.PIPELINE, succeedingInvoker()); + - assertThrows(ExitUtils.ExitException.class, () -> stateMachine.applyTransaction(newTransaction(9))); ++ assertThrows(ExitUtils.ExitException.class, ++ () -> stateMachine.applyTransaction(newTransaction(RequestType.PIPELINE, 9))); + + verify(buffer).endApplyingTransaction(); + } finally { + ExitUtils.clear(); + } + } - SCMRatisRequest request = SCMRatisRequest.of( - RequestType.FINALIZE, "finalize", new Class<?>[]{}); - private static TransactionContext newTransaction(long index) throws Exception { - SCMRatisRequest request = SCMRatisRequest.of(RequestType.PIPELINE, "op", new Class<?>[0]); - RaftProtos.StateMachineLogEntryProto smLogEntry = RaftProtos.StateMachineLogEntryProto.newBuilder() ++ private static TransactionContext newTransaction(RequestType type, long index) throws Exception { ++ // The operation name is only used by ScmInvoker#invokeLocal to pick a method to call. The invokers ++ // here are mocked and stubbed for any operation, so any non-empty name works. ++ SCMRatisRequest request = SCMRatisRequest.of(type, "op", new Class<?>[0]); + StateMachineLogEntryProto smLogEntry = StateMachineLogEntryProto.newBuilder() .setLogData(request.encode().getContent()) .build(); - RaftProtos.LogEntryProto logEntry = RaftProtos.LogEntryProto.newBuilder() + LogEntryProto logEntry = LogEntryProto.newBuilder() .setTerm(1) - .setIndex(1) + .setIndex(index) .setStateMachineLogEntry(smLogEntry) .build(); TransactionContext trx = mock(TransactionContext.class); when(trx.getStateMachineLogEntry()).thenReturn(smLogEntry); when(trx.getLogEntry()).thenReturn(logEntry); + return trx; + } - // terminate throws ExitException when system exit is disabled - assertThrows(ExitUtils.ExitException.class, - () -> stateMachine.applyTransaction(trx)); - private static SCMStateMachine newStateMachine(SCMHADBTransactionBuffer buffer, ScmInvoker<?> invoker) { ++ private static SCMStateMachine newStateMachine(SCMHADBTransactionBuffer buffer, RequestType type, ++ ScmInvoker<?> invoker) { + StorageContainerManager scm = mock(StorageContainerManager.class); + when(scm.getMetrics()).thenReturn(mock(SCMMetrics.class)); + SCMContext context = mock(SCMContext.class); + when(context.isLeader()).thenReturn(true); + when(scm.getScmContext()).thenReturn(context); + when(scm.getDatanodeProtocolServer()).thenReturn(mock(SCMDatanodeProtocolServer.class)); + when(scm.getScmSafeModeManager()).thenReturn(mock(SCMSafeModeManager.class)); + when(buffer.getLatestTrxInfo()).thenReturn(TransactionInfo.valueOf(TermIndex.valueOf(0, 0))); + SCMStateMachine stateMachine = new SCMStateMachine(scm, buffer); - stateMachine.registerInvoker(RequestType.PIPELINE, invoker); ++ stateMachine.registerInvoker(type, invoker); + return stateMachine; + } - metrics.unRegister(); + private static ScmInvoker<?> succeedingInvoker() throws Exception { + ScmInvoker<?> invoker = mock(ScmInvoker.class); + when(invoker.invokeLocal(anyString(), any())).thenReturn(Message.EMPTY); + return invoker; } } diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java index 03e220d6d62,af006c7a41e..a50c6fa7caa --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestDeadNodeHandler.java @@@ -293,12 -277,39 +277,39 @@@ public class TestDeadNodeHandler .getContainerReplicas(ContainerID.valueOf(container3.getContainerID())); assertEquals(1, container3Replicas.size()); assertEquals(datanode3, container3Replicas.iterator().next().getDatanodeDetails()); + } - //datanode will be added back to ClusterNetworkTopology if it resurrects - unhealthyToHealthyNodeHandler.onMessage(datanode1, publisher); - assertTrue( - nodeManager.getClusterNetworkTopologyMap().contains(datanode1)); - + /** + * Verifies that DeadNodeHandler skips the event when the node is no longer DEAD, for example because it - * heartbeated again and became HEALTHY_READONLY before the event was handled. ++ * heartbeated again and became HEALTHY before the event was handled. + */ + @Test + public void testDeadNodeHandlerSkipsNodeThatIsNoLongerDead( + @TempDir File tempDir) throws Exception { + DatanodeDetails datanode = MockDatanodeDetails.randomDatanodeDetails(); + String storagePath = tempDir.getPath() + .concat("/data-" + datanode.getID()); + String metaStoragePath = tempDir.getPath() + .concat("/metadata-" + datanode.getID()); + StorageReportProto storageReport = HddsTestUtils.createStorageReport( + datanode.getID(), storagePath, 100 * OzoneConsts.TB, + 10 * OzoneConsts.TB, 90 * OzoneConsts.TB, null); + MetadataStorageReportProto metaStorageReport = + HddsTestUtils.createMetadataStorageReport(metaStoragePath, + 100 * OzoneConsts.GB, 10 * OzoneConsts.GB, + 90 * OzoneConsts.GB, null); + nodeManager.register(datanode, + HddsTestUtils.createNodeReport(Arrays.asList(storageReport), + Arrays.asList(metaStorageReport)), null); + datanode = nodeManager.getNode(datanode.getID()); + - // The DEAD_NODE event was fired, but the node heartbeated again and became HEALTHY_READONLY before ++ // The DEAD_NODE event was fired, but the node heartbeated again and became HEALTHY before + // DeadNodeHandler processed it. The handler should see the current state and skip the event. - setNodeHealthState(datanode, HddsProtos.NodeState.HEALTHY_READONLY); ++ setNodeHealthState(datanode, HddsProtos.NodeState.HEALTHY); + deadNodeHandler.onMessage(datanode, publisher); + + verify(publisher, times(0)).fireEvent(SCMEvents.REPLICATION_MANAGER_NOTIFY, datanode); + verify(deletedBlockLog, times(0)).onDatanodeDead(datanode.getID()); } private void setNodeHealthState(DatanodeDetails datanode, diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java index 4578e4529df,1280ba045e6..bb4f67b8b46 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestNodeStateManager.java @@@ -17,10 -17,16 +17,15 @@@ package org.apache.hadoop.hdds.scm.node; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultLayoutVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; + import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; + import static org.mockito.ArgumentMatchers.any; + import static org.mockito.Mockito.doThrow; + import static org.mockito.Mockito.mock; + import static org.mockito.Mockito.when; import java.io.IOException; import java.util.ArrayList; @@@ -36,13 -43,17 +42,15 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.scm.container.ContainerID; import org.apache.hadoop.hdds.scm.events.SCMEvents; import org.apache.hadoop.hdds.scm.ha.SCMContext; + import org.apache.hadoop.hdds.scm.net.NetworkTopology; + import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.node.states.NodeAlreadyExistsException; import org.apache.hadoop.hdds.scm.node.states.NodeNotFoundException; -import org.apache.hadoop.hdds.scm.server.upgrade.FinalizationCheckpoint; import org.apache.hadoop.hdds.server.events.Event; import org.apache.hadoop.hdds.server.events.EventPublisher; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.ozone.container.upgrade.UpgradeUtils; -import org.apache.hadoop.ozone.upgrade.LayoutVersionManager; import org.apache.hadoop.util.Time; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@@ -56,6 -69,14 +64,8 @@@ public class TestNodeStateManager private NodeStateManager nsm; private ConfigurationSource conf; private MockEventPublisher eventPublisher; + private SCMContext scmContext; + private NetworkTopology clusterMap; - private LayoutVersionManager mockVersionManager; - private int scmSlv; - private int scmMlv; @BeforeEach public void setUp() { @@@ -75,11 -96,19 +85,10 @@@ return new char[0]; } }; -- // Make NodeStateManager behave as if SCM has completed finalization, -- // unless a test changes the value of this variable. - SCMContext scmContext = SCMContext.emptyContext(); + scmContext = SCMContext.emptyContext(); - scmContext.setFinalizationCheckpoint( - FinalizationCheckpoint.FINALIZATION_COMPLETE); eventPublisher = new MockEventPublisher(); - nsm = new NodeStateManager(conf, eventPublisher, scmContext); - scmSlv = maxLayoutVersion(); - scmMlv = maxLayoutVersion(); - mockVersionManager = mock(HDDSLayoutVersionManager.class); - when(mockVersionManager.getMetadataLayoutVersion()).thenReturn(scmMlv); - when(mockVersionManager.getSoftwareLayoutVersion()).thenReturn(scmSlv); + clusterMap = new NetworkTopologyImpl(new OzoneConfiguration()); - nsm = new NodeStateManager(conf, eventPublisher, clusterMap, mockVersionManager, scmContext); ++ nsm = new NodeStateManager(conf, eventPublisher, clusterMap, scmContext); } @Test @@@ -201,11 -230,109 +210,82 @@@ assertEquals(SCMEvents.STALE_NODE, eventPublisher.getLastEvent()); dni.updateLastHeartbeatTime(); nsm.checkNodesHealth(); - assertEquals(NodeState.HEALTHY_READONLY, nsm.getNodeStatus(dn).getHealth()); - assertEquals(SCMEvents.HEALTHY_READONLY_NODE, eventPublisher.getLastEvent()); - - // Another health check run should move the node to healthy since its - // metadata layout version matches SCM's. - nsm.checkNodesHealth(); assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); - assertEquals(SCMEvents.HEALTHY_READONLY_TO_HEALTHY_NODE, eventPublisher.getLastEvent()); + assertEquals(SCMEvents.UNHEALTHY_TO_HEALTHY_NODE, eventPublisher.getLastEvent()); eventPublisher.clearEvents(); - - // Test how node state manager handles datanodes with lower metadata - // layout version based on SCM's finalization checkpoint. - dni.updateLastKnownLayoutVersion( - UpgradeUtils.toLayoutVersionProto(scmMlv - 1, scmSlv)); - for (FinalizationCheckpoint checkpoint: FinalizationCheckpoint.values()) { - scmContext.setFinalizationCheckpoint(checkpoint); - LOG.info("Testing datanode state from current SCM finalization " + - "checkpoint: {}", checkpoint); - nsm.checkNodesHealth(); - - // Datanodes should not be moved to healthy readonly until the SCM has - // finished updating its metadata layout version as part of finalization. - if (checkpoint.hasCrossed(FinalizationCheckpoint.MLV_EQUALS_SLV)) { - assertEquals(NodeState.HEALTHY_READONLY, nsm.getNodeStatus(dn).getHealth()); - assertEquals(SCMEvents.HEALTHY_READONLY_NODE, eventPublisher.getLastEvent()); - } else { - assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); - assertNull(eventPublisher.getLastEvent()); - } - } } + @Test + public void testDeadNodeIsRemovedFromTopologyUntilItRecovers() + throws NodeAlreadyExistsException, NodeNotFoundException { + long now = Time.monotonicNow(); + long staleLimit = HddsServerUtil.getStaleNodeInterval(conf) + 1000; + long deadLimit = HddsServerUtil.getDeadNodeInterval(conf) + 1000; + + // Like SCMNodeManager, add the node to the topology before adding it to NodeStateManager. + DatanodeDetails dn = generateDatanode(); + dn.setNetworkName(dn.getUuidString()); + clusterMap.add(dn); - nsm.addNode(dn, defaultLayoutVersionProto()); ++ nsm.addNode(dn, defaultVersionProto()); + DatanodeInfo dni = nsm.getNode(dn); + String path = dn.getNetworkFullPath(); + + dni.updateLastHeartbeatTime(now - staleLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.STALE, nsm.getNodeStatus(dn).getHealth()); + assertNotNull(clusterMap.getNode(path)); + + dni.updateLastHeartbeatTime(now - deadLimit); + nsm.checkNodesHealth(); + assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth()); + assertNull(clusterMap.getNode(path)); + + dni.updateLastHeartbeatTime(); + nsm.checkNodesHealth(); - assertEquals(NodeState.HEALTHY_READONLY, nsm.getNodeStatus(dn).getHealth()); ++ assertEquals(NodeState.HEALTHY, nsm.getNodeStatus(dn).getHealth()); + assertNotNull(clusterMap.getNode(path)); + } + + @Test + public void testReregisteredNodeIsRemovedFromTopologyWhenDead() + throws NodeAlreadyExistsException, NodeNotFoundException { + DatanodeDetails dn = generateDatanode(); + dn.setNetworkName(dn.getUuidString()); + clusterMap.add(dn); - nsm.addNode(dn, defaultLayoutVersionProto()); ++ nsm.addNode(dn, defaultVersionProto()); + String path = dn.getNetworkFullPath(); + + // Re-registering with a new version or new ports replaces the DatanodeInfo with a copy that has no parent. + DatanodeDetails reregistered = new DatanodeDetails(dn); + reregistered.setParent(null); - nsm.updateNode(reregistered, defaultLayoutVersionProto()); ++ nsm.updateNode(reregistered, defaultVersionProto()); + + nsm.getNode(dn).updateLastHeartbeatTime(Time.monotonicNow() - HddsServerUtil.getDeadNodeInterval(conf) - 1000); + nsm.checkNodesHealth(); + nsm.checkNodesHealth(); + + assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth()); + assertNull(clusterMap.getNode(path)); + } + + @Test + public void testTopologyUpdateFailureDoesNotFailHealthCheck() + throws NodeAlreadyExistsException, NodeNotFoundException { + NetworkTopology failingClusterMap = mock(NetworkTopology.class); + when(failingClusterMap.contains(any())).thenReturn(true); + doThrow(new IllegalStateException("injected")).when(failingClusterMap).remove(any()); - nsm = new NodeStateManager(conf, eventPublisher, failingClusterMap, mockVersionManager, scmContext); ++ nsm = new NodeStateManager(conf, eventPublisher, failingClusterMap, scmContext); + + DatanodeDetails dn = generateDatanode(); - nsm.addNode(dn, defaultLayoutVersionProto()); ++ nsm.addNode(dn, defaultVersionProto()); + nsm.getNode(dn).updateLastHeartbeatTime(Time.monotonicNow() - HddsServerUtil.getDeadNodeInterval(conf) - 1000); + nsm.checkNodesHealth(); + nsm.checkNodesHealth(); + + assertEquals(NodeState.DEAD, nsm.getNodeStatus(dn).getHealth()); + assertEquals(SCMEvents.DEAD_NODE, eventPublisher.getLastEvent()); + } + @Test public void testNodeOpStateCanBeSet() throws NodeAlreadyExistsException, NodeNotFoundException { diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index aadc818769b,522fb17e1fb..e829de028e7 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@@ -2375,9 -2393,11 +2386,11 @@@ public class RpcClient implements Clien @Override public OzoneMultipartUploadList listMultipartUploads(String volumeName, String bucketName, String prefix, String keyMarker, String uploadIdMarker, int maxUploads) throws IOException { + verifyVolumeName(volumeName); + verifyBucketName(bucketName); OmMultipartUploadList omMultipartUploadList; - if (omVersion.compareTo(OzoneManagerVersion.S3_LIST_MULTIPART_UPLOADS_PAGINATION) >= 0) { + if (OzoneManagerVersion.S3_LIST_MULTIPART_UPLOADS_PAGINATION.isSupportedBy(omVersion)) { omMultipartUploadList = ozoneManagerClient.listMultipartUploads(volumeName, bucketName, prefix, keyMarker, uploadIdMarker, maxUploads, true); } else { diff --cc hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java index ca985fc993b,c7c4d343f6c..21a31808f06 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/rpc/TestRpcClient.java @@@ -325,18 -252,20 +328,32 @@@ public class TestRpcClient } } + @ParameterizedTest + @CsvSource({"invalid/volume, bucket1, INVALID_VOLUME_NAME", "volume1, invalid/bucket, INVALID_BUCKET_NAME"}) + public void testListMultipartUploadsValidatesNames(String volume, String bucket, ResultCodes expected) + throws Exception { + RpcClient client = createRpcClient(); + try { + OMException error = assertThrows(OMException.class, + () -> client.listMultipartUploads(volume, bucket, null, null, null, 10)); + assertThat(error.getResult()).isEqualTo(expected); + } finally { + client.close(); + } + } + + private static ServiceInfo om(OzoneManagerVersion version) { + return new ServiceInfo.Builder() + .setNodeType(HddsProtos.NodeType.OM) + .setHostname("localhost") + .setOmVersion(version) + .build(); + } + + private static ServiceInfoEx serviceInfoEx(ServiceInfo... serviceInfos) { + return new ServiceInfoEx(Arrays.asList(serviceInfos), null, null); + } + private static RpcClient createRpcClient() throws IOException { OzoneConfiguration config = new OzoneConfiguration(); return new RpcClient(config, null) { diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 5380f3ce4f0,e118ef68b41..0e457553e79 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@@ -264,13 -264,10 +264,15 @@@ public final class OmUtils case GetSnapshotInfo: case GetObjectTagging: case GetBucketTagging: + case GetBucketDeletedBytes: + return true; case GetQuotaRepairStatus: case StartQuotaRepair: + // Prepare and CancelPrepare are now no-ops, but still produce responses for compatability. + // They are marked as read-only since they don't modify state anymore. + case Prepare: + case CancelPrepare: + case QueryUpgradeStatus: case GetLifecycleConfiguration: case GetLifecycleServiceStatus: return true; diff --cc hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java index 720ba53fee0,279889d930d..06c95559b28 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestHadoopRpcOMFollowerReadFailoverProxyProvider.java @@@ -569,14 -637,16 +637,16 @@@ public class TestHadoopRpcOMFollowerRea GetKeyInfoRequest.Builder req = GetKeyInfoRequest.newBuilder() .setKeyArgs(keyArgs); - OMRequest omRequest = OMRequest.newBuilder() + OMRequest.Builder omRequest = OMRequest.newBuilder() - .setVersion(ClientVersion.CURRENT_VERSION) + .setVersion(ClientVersion.CURRENT.serialize()) .setClientId(ClientId.randomId().toString()) .setCmdType(Type.GetKeyInfo) - .setGetKeyInfoRequest(req) - .build(); + .setGetKeyInfoRequest(req); + if (readConsistency != null) { + omRequest.setReadConsistencyHint(readConsistency.getHint()); + } - client.submitRequest(null, omRequest); + client.submitRequest(null, omRequest.build()); } private void assertHandledBy(int omNodeIdx) { diff --cc hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java index 0a0e45dd37d,79d524f004c..1ee4279c287 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/protocolPB/TestS3GrpcOmTransport.java @@@ -373,6 -374,124 +374,124 @@@ public class TestS3GrpcOmTransport } @Test + public void testLeaderOnlyReadOverridesDefaultFollowerRead() throws Exception { + conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, true); + configureHaOmService("om0", "om1"); + + AtomicInteger leaderRequestCount = new AtomicInteger(); + AtomicInteger followerRequestCount = new AtomicInteger(); + AtomicReference<OMRequest> leaderRequest = new AtomicReference<>(); + + client = new GrpcOmTransport(conf, ugi, omServiceId); + client.startClient("om0", createNodeChannel("om0", + leaderRequestCount, leaderRequest)); + client.startClient("om1", createNodeChannel("om1", + followerRequestCount, new AtomicReference<>())); + client.changeLeaderProxyForTest("om0"); + client.changeFollowerReadInitialProxy("om1"); + + client.submitRequest(OMRequest.newBuilder() + .setCmdType(Type.ListVolume) - .setVersion(CURRENT_VERSION) ++ .setVersion(CURRENT.serialize()) + .setClientId("test") + .setS3Authentication(S3Authentication.newBuilder() + .setAccessId("access-id") + .setSignature("signature") + .setStringToSign("string-to-sign")) + .setReadConsistencyHint(ReadConsistencyHint.newBuilder() + .setReadConsistency(ReadConsistencyProto.LINEARIZABLE_LEADER_ONLY)) + .build()); + + assertEquals(1, leaderRequestCount.get()); + assertEquals(0, followerRequestCount.get()); + assertEquals(ReadConsistencyProto.LINEARIZABLE_LEADER_ONLY, + leaderRequest.get().getReadConsistencyHint().getReadConsistency()); + + client.submitRequest(OMRequest.newBuilder() + .setCmdType(Type.ListVolume) - .setVersion(CURRENT_VERSION) ++ .setVersion(CURRENT.serialize()) + .setClientId("test") + .setReadConsistencyHint(ReadConsistencyHint.newBuilder() + .setReadConsistency(ReadConsistencyProto.LINEARIZABLE_ALLOW_FOLLOWER)) + .build()); + + assertEquals(1, followerRequestCount.get()); + } + + @Test + public void testLocalLeaseReadAvoidsKnownLeader() throws Exception { + conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, true); + configureHaOmService("om0", "om1"); + + AtomicInteger leaderRequestCount = new AtomicInteger(); + AtomicInteger followerRequestCount = new AtomicInteger(); + + client = new GrpcOmTransport(conf, ugi, omServiceId); + client.startClient("om0", createNodeChannel("om0", + leaderRequestCount, new AtomicReference<>())); + client.startClient("om1", createNodeChannel("om1", + followerRequestCount, new AtomicReference<>())); + client.changeLeaderProxyForTest("om0"); + client.changeFollowerReadInitialProxy("om0"); + + client.submitRequest(OMRequest.newBuilder() + .setCmdType(Type.ListVolume) - .setVersion(CURRENT_VERSION) ++ .setVersion(CURRENT.serialize()) + .setClientId("test") + .setS3Authentication(S3Authentication.newBuilder() + .setAccessId("access-id") + .setSignature("signature") + .setStringToSign("string-to-sign") + .setSessionToken("session-token")) + .setReadConsistencyHint(ReadConsistencyHint.newBuilder() + .setReadConsistency(ReadConsistencyProto.LOCAL_LEASE)) + .build()); + + assertEquals(0, leaderRequestCount.get()); + assertEquals(1, followerRequestCount.get()); + } + + @Test + public void testExplicitFollowerReadWhenDisabledByDefault() throws Exception { + conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, false); + configureHaOmService("om0", "om1"); + + AtomicInteger leaderRequestCount = new AtomicInteger(); + AtomicInteger followerRequestCount = new AtomicInteger(); + AtomicReference<OMRequest> leaderRequest = new AtomicReference<>(); + AtomicReference<OMRequest> followerRequest = new AtomicReference<>(); + + client = new GrpcOmTransport(conf, ugi, omServiceId); + client.startClient("om0", createNodeChannel("om0", + leaderRequestCount, leaderRequest)); + client.startClient("om1", createNodeChannel("om1", + followerRequestCount, followerRequest)); + client.changeLeaderProxyForTest("om0"); + client.changeFollowerReadInitialProxy("om1"); + + OMRequest request = OMRequest.newBuilder() + .setCmdType(Type.ListVolume) - .setVersion(CURRENT_VERSION) ++ .setVersion(CURRENT.serialize()) + .setClientId("test") + .build(); + client.submitRequest(request); + + assertEquals(1, leaderRequestCount.get()); + assertEquals(0, followerRequestCount.get()); + assertEquals(ReadConsistencyProto.DEFAULT, + leaderRequest.get().getReadConsistencyHint().getReadConsistency()); + + client.submitRequest(request.toBuilder() + .setReadConsistencyHint(ReadConsistencyHint.newBuilder() + .setReadConsistency(ReadConsistencyProto.LOCAL_LEASE)) + .build()); + + assertEquals(1, followerRequestCount.get()); + assertEquals(ReadConsistencyProto.LOCAL_LEASE, + followerRequest.get().getReadConsistencyHint().getReadConsistency()); + } + + @Test public void testFollowerReadFallsBackToLeaderOnNotLeaderException() throws Exception { conf.setBoolean(OzoneConfigKeys.OZONE_CLIENT_FOLLOWER_READ_ENABLED_KEY, true); configureHaOmService("om0", "om1"); diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java index 278fda0f5ee,6a431e477b9..ca56caa2191 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockDataStreamOutput.java @@@ -352,12 -358,11 +360,12 @@@ public class TestBlockDataStreamOutput } } + @Unhealthy("Requires HDDS-16044 to finish implementing datanode version passing to client.") @ParameterizedTest @MethodSource("clientParameters") - public void testDatanodeVersion(boolean flushDelay, boolean putBlockOnCloseEnabled) + public void testDatanodeVersion(boolean flushDelay, boolean putBlockWithoutRaft) throws Exception { - OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockOnCloseEnabled); + OzoneClientConfig config = newClientConfig(cluster.getConf(), flushDelay, putBlockWithoutRaft); try (OzoneClient client = newClient(cluster.getConf(), config)) { // Verify all DNs internally have versions set correctly List<HddsDatanodeService> dns = cluster.getHddsDatanodes(); diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 8561b8e4aa2,a781d824a57..412e2814dfd --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@@ -102,9 -103,7 +103,8 @@@ import org.apache.hadoop.ozone.om.servi import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ToolRunner; import org.apache.ozone.test.GenericTestUtils; +import org.apache.ozone.test.tag.Unhealthy; import org.apache.ratis.server.RaftServerConfigKeys; - import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@@ -2580,60 -2562,19 +2565,67 @@@ public class TestOzoneShellHA execute(ozoneShell, args); } + /** + * Test that even though prepare commands are deprecated, they still succeed without blocking writes. + */ + @Test + public void testPrepareCommandNoOp() throws IOException { + String volumeName = "vol-prepare-test"; + String bucketName = "bucket-prepare-test"; + + // Execute prepare command via CLI + String[] args = new String[] {"om", "prepare", "--service-id", omServiceId}; + execute(ozoneAdminShell, args); + out.reset(); + + // Verify write operations still work (prepare should not block them) + // Create volume + args = new String[] {"volume", "create", "o3://" + omServiceId + OZONE_URI_DELIMITER + volumeName}; + execute(ozoneShell, args); + out.reset(); + + // Create bucket + args = new String[] {"bucket", "create", "o3://" + omServiceId + OZONE_URI_DELIMITER + volumeName + + OZONE_URI_DELIMITER + bucketName}; + execute(ozoneShell, args); + out.reset(); + + // Create key + String keyName = OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + + bucketName + OZONE_URI_DELIMITER + "testkey"; + args = new String[] {"key", "put", "o3://" + omServiceId + keyName, testFile.getPath()}; + execute(ozoneShell, args); + out.reset(); + + // Verify the key was created successfully + OzoneVolume volume = client.getObjectStore().getVolume(volumeName); + OzoneBucket bucket = volume.getBucket(bucketName); + assertNotNull(bucket.getKey("testkey"), "Key should be created successfully after prepare"); + + // Execute cancelprepare command via CLI + args = new String[] {"om", "cancelprepare", "--service-id", omServiceId}; + execute(ozoneAdminShell, args); + out.reset(); + + // Cleanup + args = new String[] {"volume", "delete", volumeName, "-r", "--yes"}; + execute(ozoneShell, args); + out.reset(); + } + - private static String getKeyProviderURI(MiniKMS kms) { - if (kms == null) { - return ""; + private static String getKeyProviderURI() { + File jks = new File(kmsDir, "kms.jks"); + return JavaKeyStoreProvider.SCHEME_NAME + "://file" + jks.toURI().getPath(); + } + + private static void seedEncryptionKey(OzoneConfiguration conf) throws Exception { + KeyProvider provider = KeyProviderFactory.get(new URI(getKeyProviderURI()), conf); + if (provider.getMetadata(ENCRYPTION_KEY_NAME) == null) { + KeyProvider.Options options = KeyProvider.options(conf); + options.setBitLength(128); + provider.createKey(ENCRYPTION_KEY_NAME, options); + provider.flush(); } - return KMSClientProvider.SCHEME_NAME + "://" + - kms.getKMSUrl().toExternalForm().replace("://", "@"); } protected MiniOzoneHAClusterImpl getCluster() { diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e2ed4e40231,43f7c6aae51..1445e1bb2bf --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -171,9 -171,7 +171,10 @@@ enum Type DeleteRevokedSTSTokens = 155; GetCallerIdentity = 156; SnapshotCount = 157; - StartFinalizeUpgrade = 158; - QueryUpgradeStatus = 159; - CompleteFinalizeUpgrade = 160; + GetBucketDeletedBytes = 158; ++ StartFinalizeUpgrade = 159; ++ QueryUpgradeStatus = 160; ++ CompleteFinalizeUpgrade = 161; } enum SafeMode { @@@ -340,9 -338,7 +341,10 @@@ message OMRequest optional UpdateAssumeRoleRequest updateAssumeRoleRequest = 157; optional GetCallerIdentityRequest getCallerIdentityRequest = 158; optional SnapshotCountRequest snapshotCountRequest = 159; - optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 160; - optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 161; - optional CompleteFinalizeUpgradeRequest completeFinalizeUpgradeRequest = 162; + optional GetBucketDeletedBytesRequest getBucketDeletedBytesRequest = 160; ++ optional StartFinalizeUpgradeRequest startFinalizeUpgradeRequest = 161; ++ optional QueryUpgradeStatusRequest queryUpgradeStatusRequest = 162; ++ optional CompleteFinalizeUpgradeRequest completeFinalizeUpgradeRequest = 163; } message OMResponse { @@@ -492,9 -489,7 +494,10 @@@ optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse = 155; optional GetCallerIdentityResponse getCallerIdentityResponse = 156; optional SnapshotCountResponse snapshotCountResponse = 157; - optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 158; - optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 159; - optional CompleteFinalizeUpgradeResponse completeFinalizeUpgradeResponse = 160; + optional GetBucketDeletedBytesResponse getBucketDeletedBytesResponse = 158; ++ optional StartFinalizeUpgradeResponse startFinalizeUpgradeResponse = 159; ++ optional QueryUpgradeStatusResponse queryUpgradeStatusResponse = 160; ++ optional CompleteFinalizeUpgradeResponse completeFinalizeUpgradeResponse = 161; } enum Status { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1d0d18235ea,9b22579ec09..45d4129dcba --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -312,9 -319,10 +319,10 @@@ import org.apache.hadoop.ozone.om.servi import org.apache.hadoop.ozone.om.service.QuotaRepairTask; import org.apache.hadoop.ozone.om.service.RevokedSTSTokenCleanupService; import org.apache.hadoop.ozone.om.snapshot.defrag.SnapshotDefragService; + import org.apache.hadoop.ozone.om.snapshot.trapped.BucketDeletedDataCalculator; import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature; -import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; -import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizer; +import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizeService; +import org.apache.hadoop.ozone.om.upgrade.OMVersionManager; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerAdminProtocolProtos.OzoneManagerAdminService; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.DBUpdatesRequest; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java index 50b1d25b30d,5e4bb9ca360..4abdc1cc683 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java @@@ -50,13 -51,12 +50,10 @@@ import org.apache.hadoop.ozone.om.lock. import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.response.DummyOMClientResponse; import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; - import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocolPB.OzoneManagerRequestHandler; import org.apache.hadoop.ozone.protocolPB.RequestHandler; - import org.apache.hadoop.ozone.security.STSSecurityUtil; - import org.apache.hadoop.ozone.security.STSTokenIdentifier; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Time; import org.apache.hadoop.util.concurrent.HadoopExecutors; import org.apache.ratis.proto.RaftProtos; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java index c5cdda6ec97,ea68fa902be..1fe0a266008 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMCompleteFinalizeUpgradeResponse.java @@@ -17,33 -17,29 +17,30 @@@ package org.apache.hadoop.ozone.om.response.upgrade; -import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; +import static org.apache.hadoop.ozone.OzoneConsts.APPARENT_VERSION_KEY; - import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE; import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; - import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Response for finalizeUpgrade request. + * Response for completing OM finalization (submitted by OMUpgradeFinalizeService). */ - @CleanupTableInfo(cleanupTables = {META_TABLE}) -public class OMFinalizeUpgradeResponse extends OMClientResponse { +public class OMCompleteFinalizeUpgradeResponse extends OMClientResponse { private static final Logger LOG = - LoggerFactory.getLogger(OMFinalizeUpgradeResponse.class); - private int layoutVersionToWrite = -1; + LoggerFactory.getLogger(OMCompleteFinalizeUpgradeResponse.class); + private int serializedApparentVersion = -1; - public OMFinalizeUpgradeResponse( + public OMCompleteFinalizeUpgradeResponse( OzoneManagerProtocolProtos.OMResponse omResponse, - int layoutVersionToWrite) { + int serializedApparentVersion) { super(omResponse); - this.layoutVersionToWrite = layoutVersionToWrite; + this.serializedApparentVersion = serializedApparentVersion; } @Override diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMStartFinalizeUpgradeResponse.java index 6687e5a6950,ea68fa902be..4f257546cbd --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMStartFinalizeUpgradeResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/upgrade/OMStartFinalizeUpgradeResponse.java @@@ -17,13 -17,11 +17,10 @@@ package org.apache.hadoop.ozone.om.response.upgrade; - import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.META_TABLE; -import static org.apache.hadoop.ozone.OzoneConsts.LAYOUT_VERSION_KEY; -- import java.io.IOException; import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; - import org.apache.hadoop.ozone.om.response.CleanupTableInfo; import org.apache.hadoop.ozone.om.response.OMClientResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.slf4j.Logger; @@@ -32,24 -30,16 +29,23 @@@ import org.slf4j.LoggerFactory /** * Response for finalizeUpgrade request. */ - @CleanupTableInfo(cleanupTables = {META_TABLE}) -public class OMFinalizeUpgradeResponse extends OMClientResponse { - private static final Logger LOG = - LoggerFactory.getLogger(OMFinalizeUpgradeResponse.class); - private int layoutVersionToWrite = -1; +public class OMStartFinalizeUpgradeResponse extends OMClientResponse { + private static final Logger LOG = LoggerFactory.getLogger(OMStartFinalizeUpgradeResponse.class); + + private final boolean finalizationNeeded; + + public OMStartFinalizeUpgradeResponse(OzoneManagerProtocolProtos.OMResponse omResponse) { + this(omResponse, true); + } - public OMFinalizeUpgradeResponse( - OzoneManagerProtocolProtos.OMResponse omResponse, - int layoutVersionToWrite) { + /** + * @param finalizationNeeded whether OM still needs to finalize. When {@code false} the finalization-in-progress + * marker is not persisted, so that initiating finalize on an already-finalized cluster does not orphan the key + * in the DB (the async {@code OMUpgradeFinalizeService} would never clear it since finalization is not needed). + */ + public OMStartFinalizeUpgradeResponse(OzoneManagerProtocolProtos.OMResponse omResponse, boolean finalizationNeeded) { super(omResponse); - this.layoutVersionToWrite = layoutVersionToWrite; + this.finalizationNeeded = finalizationNeeded; } @Override diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index aef44dd73f8,2e234ac76ec..e1795f9dd95 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@@ -60,7 -60,7 +60,8 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.protocol.proto.HddsProtos.UpgradeFinalizationStatus; import org.apache.hadoop.hdds.scm.protocolPB.OzonePBHelper; import org.apache.hadoop.hdds.utils.FaultInjector; + import org.apache.hadoop.hdds.utils.db.cache.TableCacheUpdateTracker; +import org.apache.hadoop.ozone.ClientVersion; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.om.OzoneAclUtils; import org.apache.hadoop.ozone.om.OzoneManager; @@@ -266,9 -267,15 +270,15 @@@ public class OzoneManagerRequestHandle break; case ListOpenFiles: ListOpenFilesResponse listOpenFilesResponse = listOpenFiles( - request.getListOpenFilesRequest(), request.getVersion()); + request.getListOpenFilesRequest(), clientVersion); responseBuilder.setListOpenFilesResponse(listOpenFilesResponse); break; + case GetBucketDeletedBytes: + GetBucketDeletedBytesResponse getBucketDeletedBytesResponse = + getBucketDeletedBytes(request.getGetBucketDeletedBytesRequest()); + responseBuilder.setGetBucketDeletedBytesResponse( + getBucketDeletedBytesResponse); + break; case ServiceList: ServiceListResponse serviceListResponse = getServiceList( request.getServiceListRequest()); diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java index e2b66ffa228,c5640b92c01..7a0657828af --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/ratis/TestOzoneManagerStateMachine.java @@@ -74,6 -87,10 +86,7 @@@ import org.apache.hadoop.ozone.protocol import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareRequest; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareRequestArgs; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse.PrepareStatus; + import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.S3Authentication; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.UserInfo; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
