This is an automated email from the ASF dual-hosted git repository. errose28 pushed a commit to branch HDDS-14496-zdu in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e16d646d29bda26bd5da9d98ba9c2e828348dfac Merge: 98c55fe4905 6ac2289aafd Author: Ethan Rose <[email protected]> AuthorDate: Thu Jul 16 16:51:04 2026 -0400 Merge branch 'master' into HDDS-14496-zdu-master-3 * master: (53 commits) HDDS-15876. Remove ozone-csi (#10780) HDDS-15508. Add 2.2.0 to compatibility/upgrade test (#10782) HDDS-15867. [JDK25] Bump byte-buddy to 1.18.11 (#10776) HDDS-15809. Validate x-amz-copy-source-range on UploadPartCopy (#10711). HDDS-15678. OFS isDirectory/isFile should not trigger pipeline refresh or return block locations (#10741) HDDS-15171. Add available space check on follower during bootstrap. (#10185) HDDS-14043. Fix ls -e UnsupportedOperationException on ofs/o3fs (#10209) HDDS-15709. Replace random name suffix with uniqueObjectName (#10773) HDDS-15849. Force channel shutdown in XceiverClientGrpc.close (#10747) HDDS-15765. Fail fast when --node-id is omitted for OM compact/defrag on an HA cluster (#10699) HDDS-14989. Delay follower SCM DN server start until Ratis log catch-up. (#10617). HDDS-15456. Add SCM DB lookup and identify orphan(wrt SCM) and deleted-but-present containers (#10547) HDDS-15812. Rename test utils for naming convention (#10766) HDDS-15600. Fix ListObjects response for encoding-type and empty delimiter (#10586). HDDS-15656. Fix non-atomic data size accumulation in ContainerBalancer move callback (#10708) HDDS-15667. `containerbalancer status`: show stop reason and iteration details after stop (#10681) HDDS-15823. Fix flaky TestFailureHandlingByClient#testContainerExclusionWithClosedContainerException (#10721) HDDS-15837. Add health-aware container ID listing in SCM (#10736) HDDS-15851. Return 403 for expired or out-of-range pre-signed S3 URLs (#10749) HDDS-15836. Add Archiver.appendFile for incremental TAR writes (#10735) ... Conflicts: hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java hadoop-ozone/dist/src/main/compose/upgrade/test.sh hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java .github/dependabot.yml | 56 +- .github/workflows/repeat-acceptance.yml | 2 +- .mvn/extensions.xml | 2 +- .run/CsiServer.run.xml | 33 - .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 40 +- .../hdds/scm/storage/StreamBlockInputStream.java | 9 +- .../TestXceiverClientManagerCloseContention.java | 161 +++ .../scm/storage/TestStreamBlockInputStream.java | 156 +++ hadoop-hdds/common/src/main/conf/ozone-env.sh | 11 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 1 - .../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 + .../common/src/main/resources/ozone-default.xml | 23 + .../commandhandler/DeleteBlocksCommandHandler.java | 7 + .../TestDeleteBlocksCommandHandler.java | 108 ++ ...achine.java => ContainerStateMachineTests.java} | 4 +- .../ratis/TestContainerStateMachineFollower.java | 2 +- .../ratis/TestContainerStateMachineLeader.java | 2 +- .../common/volume/TestStorageVolumeChecker.java | 16 +- ...ersAbstract.java => ContainerScannerTests.java} | 2 +- .../TestBackgroundContainerDataScanner.java | 2 +- .../TestBackgroundContainerMetadataScanner.java | 2 +- .../ozoneimpl/TestOnDemandContainerScanner.java | 2 +- .../docs/content/design/efficient-snapdiff.md | 254 ++++ hadoop-hdds/docs/content/interface/CSI.md | 93 -- hadoop-hdds/docs/content/interface/CSI.png | Bin 27210 -> 0 bytes hadoop-hdds/docs/content/interface/CSI.zh.md | 92 -- hadoop-hdds/docs/content/tools/Admin.md | 4 +- hadoop-hdds/docs/content/tools/Repair.md | 4 +- .../{TestCoderBase.java => CoderTests.java} | 2 +- ...estRSRawCoderBase.java => RSRawCoderTests.java} | 4 +- .../{TestRawCoderBase.java => RawCoderTests.java} | 6 +- .../erasurecode/rawcoder/TestDummyRawCoder.java | 2 +- .../erasurecode/rawcoder/TestNativeRSRawCoder.java | 2 +- .../rawcoder/TestNativeXORRawCoder.java | 2 +- .../ozone/erasurecode/rawcoder/TestRSRawCoder.java | 2 +- .../erasurecode/rawcoder/TestXORRawCoder.java | 2 +- ...tXORRawCoderBase.java => XORRawCoderTests.java} | 4 +- .../org/apache/hadoop/hdds/utils/Archiver.java | 76 + .../hadoop/hdds/utils/DBCheckpointServlet.java | 10 +- .../org/apache/hadoop/hdds/utils/TestArchiver.java | 40 + .../src/main/proto/ScmAdminProtocol.proto | 3 + .../interface-admin/src/main/resources/proto.lock | 18 + .../utils/{TestUtils.java => RocksTestUtils.java} | 4 +- .../utils/db/TestManagedRawSSTFileIterator.java | 4 +- .../hadoop/hdds/utils/db/TestSstFileSetReader.java | 6 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 9 +- .../scm/block/DeletedBlockLogStateManagerImpl.java | 6 +- .../SCMDeletedBlockTransactionStatusManager.java | 133 +- .../container/AbstractContainerReportHandler.java | 26 +- .../hdds/scm/container/ContainerManager.java | 4 +- .../hdds/scm/container/ContainerManagerImpl.java | 5 +- .../hdds/scm/container/ContainerStateManager.java | 5 +- .../scm/container/ContainerStateManagerImpl.java | 5 +- .../scm/container/balancer/ContainerBalancer.java | 28 +- .../balancer/ContainerBalancerStatusInfo.java | 43 +- .../balancer/ContainerBalancerStopReason.java | 63 + .../container/balancer/ContainerBalancerTask.java | 143 +- .../scm/container/states/ContainerStateMap.java | 49 +- .../apache/hadoop/hdds/scm/ha/SCMRatisServer.java | 5 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 135 +- .../apache/hadoop/hdds/scm/ha/StatefulService.java | 13 +- .../hdds/scm/ha/StatefulServiceDefinition.java} | 38 +- .../ha/invoker/ContainerStateManagerInvoker.java | 83 +- .../hdds/scm/security/RootCARotationManager.java | 6 +- .../hdds/scm/server/SCMClientProtocolServer.java | 15 +- .../hdds/scm/server/StorageContainerManager.java | 9 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 +- .../scm/container/TestContainerStateManager.java | 2 +- .../container/balancer/TestContainerBalancer.java | 43 + .../balancer/TestContainerBalancerStatusInfo.java | 66 + .../balancer/TestContainerBalancerTask.java | 58 + ...andler.java => MisReplicationHandlerTests.java} | 2 +- .../replication/TestECMisReplicationHandler.java | 2 +- .../TestRatisMisReplicationHandler.java | 2 +- .../container/states/TestContainerStateMap.java | 102 +- .../hdds/scm/node/TestContainerPlacement.java | 10 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 51 +- .../hadoop/hdds/scm/node/TestSCMNodeMetrics.java | 5 +- .../scm/cli/ContainerBalancerStatusSubcommand.java | 133 +- .../ozone/admin/om/snapshot/DefragSubCommand.java | 11 +- .../datanode/TestContainerBalancerSubCommand.java | 242 +++- .../admin/om/snapshot/TestDefragSubCommand.java | 22 + .../container/analyze/AnalyzeSubcommand.java | 234 +++- .../analyze/ScmContainerMetadataReader.java | 155 ++ .../apache/hadoop/ozone/debug/ldb/DBScanner.java | 51 +- .../ozone/debug/replicas/ReplicasVerify.java | 112 +- .../analyze/ContainerAnalyzeTestHelper.java | 37 + .../container/analyze/TestAnalyzeSubcommand.java | 188 ++- .../analyze/TestScmContainerMetadataReader.java | 100 ++ .../apache/hadoop/ozone/repair/om/CompactOMDB.java | 11 +- .../hadoop/ozone/repair/om/TestCompactOMDB.java | 12 + .../apache/hadoop/ozone/client/OzoneBucket.java | 16 + .../ozone/client/protocol/ClientProtocol.java | 22 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 3 +- .../hadoop/ozone/client/TestOzoneBucket.java | 72 + .../client/TestRpcClientGetFileStatusHeadOp.java | 93 ++ .../java/org/apache/hadoop/ozone/OzoneAcl.java | 22 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 12 + .../hadoop/ozone/om/helpers/AclListBuilder.java | 7 +- .../hadoop/ozone/om/helpers/OmDirectoryInfo.java | 4 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 74 +- .../ozone/om/helpers/OmMultipartAbortInfo.java | 42 +- .../ozone/om/helpers/OmMultipartKeyInfo.java | 17 +- .../ozone/om/helpers/OmMultipartPartInfo.java | 22 + .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 55 +- .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 44 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 1 + .../ozone/security/acl/IAccessAuthorizer.java | 16 +- .../csi/dev-support/findbugsExcludeFile.xml | 22 - hadoop-ozone/csi/pom.xml | 290 ---- .../apache/hadoop/ozone/csi/ControllerService.java | 116 -- .../org/apache/hadoop/ozone/csi/CsiServer.java | 177 --- .../apache/hadoop/ozone/csi/IdentityService.java | 72 - .../org/apache/hadoop/ozone/csi/NodeService.java | 148 -- .../org/apache/hadoop/ozone/csi/package-info.java | 21 - hadoop-ozone/csi/src/main/proto/csi.proto | 1323 ----------------- hadoop-ozone/csi/src/main/resources/proto.lock | 1479 -------------------- .../dev-support/intellij/ozone-site-ha.xml | 10 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 - hadoop-ozone/dist/pom.xml | 5 - .../dist/src/main/compose/common/replicas-test.sh | 2 + hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 20 - .../src/main/compose/ozone-csi/docker-compose.yaml | 64 - .../dist/src/main/compose/ozone-csi/docker-config | 44 - .../dist/src/main/compose/ozone-csi/test.sh | 31 - .../main/compose/ozonesecure-ha/s3-haproxy.yaml | 3 + hadoop-ozone/dist/src/main/compose/testlib.sh | 13 + hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 11 +- .../dist/src/main/compose/xcompat/clients.yaml | 8 +- hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +- .../k8s/definitions/ozone-csi/csi-controller.yaml | 53 - .../main/k8s/definitions/ozone-csi/csi-crd.yaml | 21 - .../main/k8s/definitions/ozone-csi/csi-node.yaml | 95 -- .../main/k8s/definitions/ozone-csi/csi-rbac.yaml | 66 - .../definitions/ozone-csi/csi-storageclass.yaml | 20 - .../k8s/definitions/ozone-csi/definitions/csi.yaml | 28 - .../k8s/definitions/test-webserver/flekszible.yaml | 16 - .../test-webserver/webserver-deployment.yaml | 50 - .../test-webserver/webserver-service.yaml | 29 - .../test-webserver/webserver-volume.yaml | 29 - .../src/main/k8s/examples/ozone-dev/Flekszible | 7 - .../k8s/examples/ozone-dev/config-configmap.yaml | 6 +- .../examples/ozone-dev/csi/csi-node-daemonset.yaml | 97 -- .../ozone-dev/csi/csi-ozone-clusterrole.yaml | 98 -- .../csi/csi-ozone-clusterrolebinding.yaml | 28 - .../ozone-dev/csi/csi-ozone-serviceaccount.yaml | 21 - .../ozone-dev/csi/csi-provisioner-deployment.yaml | 54 - .../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 - .../examples/ozone-dev/csi/ozone-storageclass.yaml | 21 - .../ozone-csi-test-webserver-deployment.yaml | 50 - ...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 - .../pv-test/ozone-csi-test-webserver-service.yaml | 29 - .../dist/src/main/k8s/examples/ozone/Flekszible | 5 - .../main/k8s/examples/ozone/config-configmap.yaml | 6 +- .../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 97 -- .../examples/ozone/csi/csi-ozone-clusterrole.yaml | 98 -- .../ozone/csi/csi-ozone-clusterrolebinding.yaml | 28 - .../ozone/csi/csi-ozone-serviceaccount.yaml | 21 - .../ozone/csi/csi-provisioner-deployment.yaml | 54 - .../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 - .../k8s/examples/ozone/csi/ozone-storageclass.yaml | 21 - .../ozone-csi-test-webserver-deployment.yaml | 50 - ...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 - .../pv-test/ozone-csi-test-webserver-service.yaml | 29 - hadoop-ozone/dist/src/main/license/jar-report.txt | 2 - .../src/main/smoketest/balancer/testBalancer.robot | 2 +- hadoop-ozone/dist/src/main/smoketest/csi.robot | 34 - .../src/main/smoketest/s3/MultipartUpload.robot | 15 +- hadoop-ozone/dist/src/shell/ozone/ozone | 7 - .../dist/src/shell/ozone/ozone-functions.sh | 2 +- .../AbstractTestStorageDistributionEndpoint.java | 14 +- ...ndpointUtil.java => ReconEndpointTestUtil.java} | 6 +- .../ozone/recon/TestReconAndAdminContainerCLI.java | 36 +- hadoop-ozone/integration-test-s3/pom.xml | 7 +- .../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 59 + .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 169 ++- .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 239 +++- hadoop-ozone/integration-test/pom.xml | 5 - ...eldsBase.java => ConfigurationFieldsTests.java} | 8 +- .../fs/ozone/AbstractOzoneFileSystemTest.java | 44 + .../ozone/AbstractRootedOzoneFileSystemTest.java | 89 +- .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 4 +- .../fs/ozone/TestOFSIsDirectoryBenchmark.java | 153 ++ .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 6 +- .../hdds/scm/TestContainerReportWithKeys.java | 7 +- .../TestSCMFollowerCatchupWithContainerReport.java | 336 +++++ .../TestReplicationManagerIntegration.java | 8 +- .../node/{TestNodeUtil.java => NodeTestUtil.java} | 4 +- .../scm/node/TestDecommissionAndMaintenance.java | 16 +- .../hdds/scm/safemode/TestSafeModeSCMHA.java | 18 +- ...UpgradeUtils.java => HddsUpgradeTestUtils.java} | 6 +- .../TestDNDataDistributionFinalization.java | 4 +- .../TestScmDataDistributionFinalization.java | 14 +- .../hadoop/hdds/upgrade/TestScmHAFinalization.java | 18 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 9 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +- .../ozone/client/rpc/OzoneRpcClientTests.java | 25 +- .../client/rpc/TestFailureHandlingByClient.java | 13 +- .../rpc/TestOzoneRpcClientForAclAuditLog.java | 6 +- ...tInputStreamBase.java => InputStreamTests.java} | 2 +- .../client/rpc/read/TestChunkInputStream.java | 12 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 30 +- .../rpc/read/TestStreamBlockInputStream.java | 12 +- .../ozone/client/rpc/read/TestStreamRead.java | 6 +- .../rpc/read/TestStreamReadDatanodeFailover.java | 4 +- ...estDuplicateContainerDirScannerIntegration.java | 1 + .../ozone/dn/ratis/TestDnRatisLogParser.java | 4 + ....java => ContainerScannerIntegrationTests.java} | 2 +- ...tBackgroundContainerDataScannerIntegration.java | 2 +- ...kgroundContainerMetadataScannerIntegration.java | 2 +- .../TestOnDemandContainerScannerIntegration.java | 2 +- ...estDataValidate.java => DataValidateTests.java} | 2 +- .../freon/TestDataValidateWithDummyContainers.java | 2 +- .../TestDataValidateWithSafeByteOperations.java | 2 +- .../TestDataValidateWithUnsafeByteOperations.java | 2 +- .../ozone/local/TestLocalOzoneClusterRuntime.java | 136 ++ .../ozone/om/AbstractOzoneManagerHATest.java | 11 +- .../om/{TestBucket.java => BucketForTesting.java} | 16 +- ...d.java => OzoneManagerHAFollowerReadTests.java} | 2 +- ...zoneManagerHA.java => OzoneManagerHATests.java} | 2 +- .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 8 +- .../hadoop/ozone/om/TestListKeysWithFSO.java | 12 +- .../apache/hadoop/ozone/om/TestOMBootstrap.java | 5 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 35 +- .../TestOMDbCheckpointServletInodeBasedXfer.java | 80 +- .../ozone/om/TestOMRatisSnapshotTransfer.java | 5 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 13 +- ...stOzoneManagerHAFollowerReadWithAllRunning.java | 9 +- ...OzoneManagerHAFollowerReadWithStoppedNodes.java | 2 +- .../ozone/om/TestOzoneManagerHAWithAllRunning.java | 15 +- .../om/TestOzoneManagerHAWithStoppedNodes.java | 11 +- .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +- .../hadoop/ozone/om/TestOzoneManagerRestart.java | 20 +- .../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 8 +- .../ozone/om/service/TestBlockDeletionService.java | 4 +- ...TestSnapshotDeletingServiceIntegrationTest.java | 6 +- .../{TestOmSnapshot.java => OmSnapshotTests.java} | 47 +- .../ozone/om/snapshot/TestOMDBCheckpointUtils.java | 17 + .../ozone/om/snapshot/TestOmSnapshotDisabled.java | 8 +- .../om/snapshot/TestOmSnapshotDisabledRestart.java | 8 +- .../snapshot/TestOmSnapshotFsoWithNativeLib.java | 2 +- .../TestOmSnapshotFsoWithoutNativeLib.java | 2 +- ...apshotFsoWithoutNativeLibWithLinkedBuckets.java | 2 +- .../om/snapshot/TestOmSnapshotObjectStore.java | 2 +- .../TestOmSnapshotWithoutBucketLinkingLegacy.java | 2 +- .../om/snapshot/TestOzoneManagerHASnapshot.java | 25 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 5 +- .../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +- ...skBalancerDuringDecommissionAndMaintenance.java | 4 +- .../ozone/shell/TestOzoneDebugReplicasVerify.java | 136 ++ .../src/main/proto/OmClientProtocol.proto | 4 + .../ozone/om/helpers/TestOmKeyInfoCodec.java | 132 +- .../om/helpers/TestRepeatedOmKeyInfoCodec.java | 109 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 36 + .../hadoop/ozone/om/OMDBCheckpointServlet.java | 65 +- .../om/OMDBCheckpointServletInodeBasedXfer.java | 26 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 4 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +- .../hadoop/ozone/om/codec/OMDBDefinition.java | 23 +- .../om/ratis_snapshot/OmRatisSnapshotProvider.java | 300 +++- .../hadoop/ozone/om/request/OMClientRequest.java | 39 +- .../om/request/key/OMAllocateBlockRequest.java | 4 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 3 - .../om/request/key/OMKeyCommitRequestWithFSO.java | 4 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 2 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 16 +- .../S3ExpiredMultipartUploadsAbortRequest.java | 42 +- .../multipart/S3MultipartUploadAbortRequest.java | 42 +- .../S3MultipartUploadAbortRequestWithFSO.java | 9 +- .../S3MultipartUploadCommitPartRequest.java | 74 +- .../S3MultipartUploadCommitPartRequestWithFSO.java | 8 +- .../S3MultipartUploadCompleteRequest.java | 78 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 6 +- .../om/request/util/OMMultipartUploadUtils.java | 102 ++ .../AbstractS3MultipartAbortResponse.java | 53 +- .../S3ExpiredMultipartUploadsAbortResponse.java | 3 +- .../multipart/S3MultipartUploadAbortResponse.java | 30 +- .../S3MultipartUploadAbortResponseWithFSO.java | 14 +- .../S3MultipartUploadCommitPartResponse.java | 23 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 8 +- .../S3MultipartUploadCompleteResponse.java | 15 +- .../S3MultipartUploadCompleteResponseWithFSO.java | 10 +- .../ozone/om/snapshot/OMDBCheckpointUtils.java | 74 +- .../ozone/om/snapshot/SnapshotDiffValueParser.java | 359 +++++ .../protocolPB/OzoneManagerRequestHandler.java | 13 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 33 + .../TestOmRatisSnapshotProvider.java | 59 + .../TestOMClientRequestUserInfoFallback.java | 106 -- .../om/request/key/TestOMKeyCommitRequest.java | 2 - .../TestS3MultipartUploadCommitPartRequest.java | 355 +++++ .../s3/multipart/S3MultipartResponseTests.java | 10 +- .../TestS3MultipartUploadAbortResponseWithFSO.java | 4 +- .../om/snapshot/TestSnapshotDiffValueParser.java | 249 ++++ .../protocolPB/TestOzoneManagerRequestHandler.java | 83 ++ .../fs/ozone/BasicOzoneClientAdapterImpl.java | 20 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 86 ++ .../ozone/BasicRootedOzoneClientAdapterImpl.java | 61 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 67 +- .../apache/hadoop/fs/ozone/FileStatusAdapter.java | 20 + .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 11 + .../TestBasicRootedOzoneClientAdapterHeadOp.java | 176 +++ .../fs/ozone/TestRootedOzoneFileSystemHeadOp.java | 174 +++ .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 + .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 + hadoop-ozone/pom.xml | 1 - .../webapps/recon/ozone-recon-web/.eslintignore | 26 - .../recon/ozone-recon-web/eslint.config.mjs | 109 ++ .../webapps/recon/ozone-recon-web/package.json | 15 +- .../webapps/recon/ozone-recon-web/pnpm-lock.yaml | 1111 +++++++-------- .../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 3 +- .../ozone/s3/commontypes/RequestParameters.java | 11 + .../hadoop/ozone/s3/endpoint/BucketAclHandler.java | 7 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +- ...CompleteMultipartUploadRequestUnmarshaller.java | 5 +- .../ozone/s3/endpoint/ListBucketResponse.java | 13 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 37 +- .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 41 +- .../ozone/s3/signature/AWSSignatureProcessor.java | 7 +- .../signature/AccessDeniedResourceException.java | 12 +- .../s3/signature/AuthorizationV4QueryParser.java | 7 +- .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 3 + .../org/apache/hadoop/ozone/s3/util/S3Utils.java | 11 + .../hadoop/ozone/client/ClientProtocolStub.java | 3 +- .../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +- .../hadoop/ozone/s3/endpoint/TestBucketList.java | 38 +- ...CompleteMultipartUploadRequestUnmarshaller.java | 4 +- .../s3/endpoint/TestMultipartUploadComplete.java | 17 + .../hadoop/ozone/s3/endpoint/TestRootList.java | 94 ++ .../ozone/s3/metrics/TestS3GatewayMetrics.java | 12 +- .../s3/signature/TestAWSSignatureProcessor.java | 37 + .../signature/TestAuthorizationV4QueryParser.java | 16 +- hadoop-ozone/tools/pom.xml | 12 + .../hadoop/ozone/local/LocalOzoneCluster.java | 294 +++- .../hadoop/ozone/local/TestLocalOzoneCluster.java | 9 +- pom.xml | 32 +- 342 files changed, 9512 insertions(+), 7428 deletions(-) diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java index 289b5db922a,a0f1c1bf362..8b0ea537d4e --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java @@@ -38,9 -38,9 +38,11 @@@ import static org.apache.hadoop.hdds.sc import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND; import static org.apache.hadoop.hdds.scm.events.SCMEvents.DATANODE_COMMAND_COUNT_UPDATED; import static org.apache.hadoop.hdds.scm.events.SCMEvents.NEW_NODE; -import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toLayoutVersionProto; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.defaultVersionProto; +import static org.apache.hadoop.ozone.container.upgrade.UpgradeUtils.toVersionProto; + import static org.apache.ozone.test.MetricsAsserts.getLongCounter; + import static org.apache.ozone.test.MetricsAsserts.getMetrics; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java index bfce2f29097,2db0df2db61..8235e18fd05 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeMetrics.java @@@ -17,8 -17,7 +17,7 @@@ package org.apache.hadoop.hdds.scm.node; - import static java.lang.Thread.sleep; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; +import static org.apache.hadoop.hdds.scm.upgrade.ScmUpgradeTestUtils.mockVersionManager; import static org.apache.ozone.test.MetricsAsserts.assertGauge; import static org.apache.ozone.test.MetricsAsserts.getLongCounter; import static org.apache.ozone.test.MetricsAsserts.getMetrics; @@@ -38,9 -39,10 +37,10 @@@ import org.apache.hadoop.hdds.scm.HddsT import org.apache.hadoop.hdds.scm.ha.SCMContext; import org.apache.hadoop.hdds.scm.net.NetworkTopologyImpl; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; +import org.apache.hadoop.hdds.scm.server.upgrade.ScmVersionManager; import org.apache.hadoop.hdds.server.events.EventQueue; -import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; import org.apache.hadoop.metrics2.MetricsRecordBuilder; + import org.apache.ozone.test.GenericTestUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@@ -238,8 -244,10 +238,9 @@@ public class TestSCMNodeMetrics assertGauge("TotalFilesystemAvailable", 150L, getMetrics(SCMNodeMetrics.class.getSimpleName())); nodeManager.processHeartbeat(registeredDatanode); - sleep(4000); + GenericTestUtils.waitFor( + () -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1, 100, 5000); metricsSource = getMetrics(SCMNodeMetrics.SOURCE_NAME); - assertGauge("InServiceHealthyReadonlyNodes", 0, metricsSource); assertGauge("InServiceHealthyNodes", 1, metricsSource); } diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 77420a88216,ab4da4badd9..ca911f895c7 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@@ -134,7 -143,7 +143,7 @@@ public final class OmKeyInfo extends Wi return new DelegatedCodec<>( Proto2Codec.get(KeyInfo.getDefaultInstance()), OmKeyInfo::getFromProtobuf, - k -> k.getProtobuf(true, ClientVersion.CURRENT.serialize()), - k -> k.getProtobuf(true, ClientVersion.CURRENT_VERSION, isOpenKey), ++ k -> k.getProtobuf(true, ClientVersion.CURRENT.serialize(), isOpenKey), OmKeyInfo.class); } diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java index 2b882c55b7b,cf37681a793..39a3497aa1d --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java @@@ -55,7 -60,7 +60,7 @@@ public class RepeatedOmKeyInfo implemen return new DelegatedCodec<>( Proto2Codec.get(RepeatedKeyInfo.getDefaultInstance()), RepeatedOmKeyInfo::getFromProto, - k -> k.getProto(ignorePipeline, ClientVersion.CURRENT.serialize()), - k -> k.getProto(ignorePipeline, ClientVersion.CURRENT_VERSION, isOpenKey), ++ k -> k.getProto(ignorePipeline, ClientVersion.CURRENT.serialize(), isOpenKey), RepeatedOmKeyInfo.class); } diff --cc hadoop-ozone/dist/src/main/compose/upgrade/test.sh index 3c079cd884f,e6c13160711..e9d0795db1e --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@@ -33,18 -33,16 +33,19 @@@ RESULT_DIR="$ALL_RESULT_DIR" create_res # This is the version of Ozone that should use the runner image to run the # code that was built. Other versions will pull images from docker hub. - # run_test ha non-rolling-upgrade 2.1.0 "$OZONE_CURRENT_VERSION" -run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION" ++# run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION" + # run_test ha non-rolling-upgrade 2.1.1 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION" - #run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" - #run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" + # run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" + # run_test ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 1.4.0 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 1.3.0 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 1.2.1 "$OZONE_CURRENT_VERSION" # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION" - # Rolling upgrade test, commented out for now - run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" ++# Rolling upgrade test ++run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" + generate_report "upgrade" "$ALL_RESULT_DIR" exit "$RESULT" diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java index ae7e091658c,eea584643e6..3f735828b4c --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/HddsUpgradeTestUtils.java @@@ -41,17 -56,26 +41,17 @@@ import org.slf4j.LoggerFactory /** * Helper methods for testing HDDS upgrade finalization in integration tests. */ - public final class TestHddsUpgradeUtils { + public final class HddsUpgradeTestUtils { - private static final Logger LOG = LoggerFactory.getLogger(TestHddsUpgradeUtils.class); + private static final Logger LOG = LoggerFactory.getLogger(HddsUpgradeTestUtils.class); - private TestHddsUpgradeUtils() { } - private static final ReplicationConfig RATIS_THREE = - ReplicationConfig.fromProtoTypeAndFactor(HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.THREE); - + private HddsUpgradeTestUtils() { } - public static void waitForFinalizationFromClient( - StorageContainerLocationProtocol scmClient, String clientID) - throws Exception { + public static void waitForFinalizationFromClient(StorageContainerLocationProtocol scmClient) throws Exception { LambdaTestUtils.await(60_000, 1_000, () -> { - UpgradeFinalization.Status status = scmClient - .queryUpgradeFinalizationProgress(clientID, true, true) - .status(); - LOG.info("Waiting for upgrade finalization to complete from client." + - " Current status is {}.", status); - return status == FINALIZATION_DONE || status == ALREADY_FINALIZED; + HddsProtos.UpgradeStatus status = scmClient.queryUpgradeStatus(); + LOG.info("Waiting for upgrade finalization to complete from client. Current status is:\n{}", status); + return status.getShouldFinalize(); }); } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java index 76bab76d66b,8c7daace078..a33af09e11f --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java @@@ -154,12 -167,24 +154,12 @@@ public class TestDNDataDistributionFina // Validate pre-finalization state validatePreDataDistributionFeatureState(); - // Now trigger finalization - Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - // Wait for finalization to complete - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); // Verify finalization completed - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization()); // Create more data and deletions to test post-finalization behavior String keyName3 = "testKey3"; @@@ -190,10 -215,21 +190,10 @@@ out.write(data); } bucket.deleteKey(keyName); - Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - // Wait for finalization - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); + + assertFalse(cluster.getStorageContainerManager().getVersionManager().needsFinalization()); // Verify the system can handle scenarios where pendingDeleteBlockCount // might be missing and needs recalculation diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java index 6707cd87f00,3fb1bc121af..81fb21ebcde --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java @@@ -26,9 -26,9 +26,9 @@@ import static org.apache.hadoop.hdds.Hd import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT; import static org.apache.hadoop.hdds.client.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.client.ReplicationType.RATIS; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL; import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.EMPTY_SUMMARY; - import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize; ++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmsToFinalize; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL; import static org.apache.hadoop.ozone.common.BlockGroup.SIZE_NOT_AVAILABLE; import static org.junit.jupiter.api.Assertions.assertEquals; @@@ -160,18 -191,20 +160,18 @@@ public class TestScmDataDistributionFin @Test @Flaky("HDDS-14050") public void testFinalizationEmptyClusterDataDistribution() throws Exception { - init(new OzoneConfiguration(), null, true); + init(new OzoneConfiguration()); assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( - cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); + cluster.getStorageContainerManagersList(), 0); - TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( + HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( - cluster.getHddsDatanodes(), 0, CLOSED); + cluster.getHddsDatanodes(), 0); assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) { @@@ -266,15 -299,26 +266,15 @@@ flushDBTransactionBuffer(activeSCM); assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); - finalizationFuture = Executors.newSingleThreadExecutor().submit( - () -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - LOG.info("finalization client failed. This may be expected if the" + - " test injected failures.", ex); - } - }); - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( - cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); + cluster.getStorageContainerManagersList(), 0); - TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( + HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( - cluster.getHddsDatanodes(), 0, CLOSED); + cluster.getHddsDatanodes(), 0); assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); for (StorageContainerManager scm: cluster.getStorageContainerManagersList()) { diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java index eff92e27589,c0e64ac4cac..802562f886a --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmHAFinalization.java @@@ -17,13 -17,21 +17,13 @@@ package org.apache.hadoop.hdds.upgrade; - import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmToFinalize; - import static org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils.waitForScmsToFinalize; -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ContainerDataProto.State.CLOSED; ++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmToFinalize; ++import static org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils.waitForScmsToFinalize; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.List; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.stream.Stream; +import org.apache.hadoop.hdds.HDDSVersion; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.scm.ScmConfigKeys; @@@ -90,54 -126,118 +90,54 @@@ public class TestScmHAFinalization } } - /** - * Argument supplier for parameterized tests. - */ - public static Stream<Arguments> injectionPointsToTest() { - // Do not test from BEFORE_PRE_FINALIZE_UPGRADE injection point. - // Finalization will not have started so there will be no persisted state - // to resume from. - return Stream.of( - Arguments.of(UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE), - Arguments.of(UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION), - Arguments.of(UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE) - ); - } - - @ParameterizedTest - @MethodSource(METHOD_SOURCE) - public void testFinalizationWithLeaderChange( - UpgradeTestInjectionPoints haltingPoint) throws Exception { - - CountDownLatch pauseLatch = new CountDownLatch(1); - CountDownLatch unpauseLatch = new CountDownLatch(1); - init(new OzoneConfiguration(), - UpgradeTestUtils.newPausingFinalizationExecutor(haltingPoint, - pauseLatch, unpauseLatch, LOG), 0); - pauseLatch.await(); - - // Stop the leader, forcing a leader change in the middle of finalization. - // This will cause the initial client call for finalization - // to be interrupted. - StorageContainerManager oldLeaderScm = cluster.getActiveSCM(); - LOG.info("Stopping current SCM leader {} to initiate a leader change.", - oldLeaderScm.getSCMNodeId()); - cluster.shutdownStorageContainerManager(oldLeaderScm); - - // Wait for the remaining two SCMs to elect a new leader. - cluster.waitForClusterToBeReady(); - - // While finalization is paused, check its state on the remaining SCMs. - checkMidFinalizationConditions(haltingPoint, - cluster.getStorageContainerManagersList()); - - // Restart actually creates a new SCM. - // Since this SCM will be a follower, the implementation of its upgrade - // finalization executor does not matter for this test. - cluster.restartStorageContainerManager(oldLeaderScm, true); - - // Make sure the original SCM leader is not the leader anymore. - StorageContainerManager newLeaderScm = cluster.getActiveSCM(); - assertNotEquals(newLeaderScm.getSCMNodeId(), - oldLeaderScm.getSCMNodeId()); - - // Resume finalization from the new leader. - unpauseLatch.countDown(); - - // Client should complete exceptionally since the original SCM it - // requested to was restarted. - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - // Make sure old leader has caught up and all SCMs have finalized. - waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - - HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( - cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); - HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( - cluster.getHddsDatanodes(), 0, CLOSED); - } - - @ParameterizedTest - @MethodSource(METHOD_SOURCE) - @Flaky("HDDS-8714") - public void testFinalizationWithRestart( - UpgradeTestInjectionPoints haltingPoint) throws Exception { - CountDownLatch terminateLatch = new CountDownLatch(1); - init(new OzoneConfiguration(), - UpgradeTestUtils.newTerminatingFinalizationExecutor(haltingPoint, - terminateLatch, LOG), - 0); - terminateLatch.await(); + @Test + public void testFinalizedDatanodesShutDownWithPrefinalizedScm() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setInt(SCMStorageConfig.TESTING_INIT_APPARENT_VERSION_KEY, HDDSLayoutFeature.INITIAL_VERSION.serialize()); + conf.set(ScmConfigKeys.OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT, "5s"); + conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL_DEFAULT, "1s"); - // Once upgrade finalization is stopped at the halting point, restart all - // SCMs. - LOG.info("Restarting all SCMs during upgrade finalization."); - // Restarting an SCM from mini ozone actually replaces the SCM with a new - // instance. We will use the normal upgrade finalization executor for - // these new instances, since the last one aborted at the halting point. - cluster.getSCMConfigurator() - .setUpgradeFinalizationExecutor( - new DefaultUpgradeFinalizationExecutor<>()); - List<StorageContainerManager> originalSCMs = - cluster.getStorageContainerManagers(); + MiniOzoneHAClusterImpl.Builder clusterBuilder = MiniOzoneCluster.newHABuilder(conf); + clusterBuilder.setNumOfStorageContainerManagers(NUM_SCMS) + .setNumOfActiveSCMs(NUM_SCMS) + .setSCMServiceId("scmservice") + .setNumOfOzoneManagers(1) + .setNumDatanodes(NUM_DATANODES) + .setDatanodeFactory(UniformDatanodesFactory.newBuilder() + .setApparentVersion(HDDSVersion.SOFTWARE_VERSION.serialize()) + .build()); - for (StorageContainerManager scm: originalSCMs) { - cluster.restartStorageContainerManager(scm, false); + // Prevent terminateDatanode() from calling System.exit(1) and killing the test JVM. + ExitUtil.disableSystemExit(); + LogCapturer logCapture = LogCapturer.captureLogs(RegisterEndpointTask.class); + // This starts the mini ozone cluster. + cluster = clusterBuilder.build(); + + // isStopped cannot be set to true unless a datanode was started first. + // Each datanode should be rejected since its apparent version exceeds the pre-finalized SCM's. + GenericTestUtils.waitFor( + () -> cluster.getHddsDatanodes().stream().allMatch(HddsDatanodeService::isStopped), + 500, 30_000); + + assertThat(logCapture.getOutput()).contains("SCM rejected this datanode's registration"); + for (StorageContainerManager scm : cluster.getStorageContainerManagersList()) { + assertThat(scm.getScmNodeManager().getAllNodes()).isEmpty(); } + } - checkMidFinalizationConditions(haltingPoint, - cluster.getStorageContainerManagersList()); - - // After all SCMs were restarted, finalization should resume - // automatically once a leader is elected. - cluster.waitForClusterToBeReady(); - - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - // Once the leader tells the client finalization is complete, wait for all - // followers to catch up so we can check their state. + @Test + public void testFinalization() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + init(conf, 0); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); + // Ensure all SCMs finalize, indicating the message has been propagated across them all waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( - cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); + cluster.getStorageContainerManagersList(), 0); - TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( + HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( - cluster.getHddsDatanodes(), 0, CLOSED); + cluster.getHddsDatanodes(), 0); } @Test @@@ -167,15 -267,15 +167,15 @@@ } // Wait for finalization from the client perspective. - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); // Wait for two running SCMs to finish finalization. waitForScmsToFinalize(activeScms); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( - activeScms, 0, NUM_DATANODES); + activeScms, 0); - TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( + HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( - cluster.getHddsDatanodes(), 0, CLOSED); + cluster.getHddsDatanodes(), 0); // Move SCM log index farther ahead to make sure a snapshot install // happens on the restarted SCM. @@@ -188,16 -288,90 +188,16 @@@ } cluster.startInactiveSCM(inactiveScm.getSCMNodeId()); - waitForScmToFinalize(inactiveScm); + LOG.info("Waiting for restarted SCM to finalize"); + // When the leader sends a snapshot to the follower, it should have flushed all entries to the DB, including the + // apparent versin. This means the follower should see it in the DB it receives immediately to trigger finalization. + waitForScmToFinalize(inactiveScm, true); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( - inactiveScm, 0, NUM_DATANODES); + inactiveScm, 0); // Use log to verify a snapshot was installed. - assertThat(logCapture.getOutput()).contains("New SCM snapshot " + - "received with metadata layout version"); - } - - private void waitForScmsToFinalize(Collection<StorageContainerManager> scms) - throws Exception { - for (StorageContainerManager scm: scms) { - waitForScmToFinalize(scm); - } - } - - private void waitForScmToFinalize(StorageContainerManager scm) - throws Exception { - GenericTestUtils.waitFor(() -> !scm.isInSafeMode(), 500, 5000); - GenericTestUtils.waitFor(() -> { - FinalizationCheckpoint checkpoint = - scm.getScmContext().getFinalizationCheckpoint(); - LOG.info("Waiting for SCM {} (leader? {}) to finalize. Current " + - "finalization checkpoint is {}", - scm.getSCMNodeId(), scm.checkLeader(), checkpoint); - return checkpoint.hasCrossed( - FinalizationCheckpoint.FINALIZATION_COMPLETE); - }, 2_000, 60_000); - } - - private void checkMidFinalizationConditions( - UpgradeTestInjectionPoints haltingPoint, - List<StorageContainerManager> scms) { - - // Ratis only makes sure that the Leader has processed the finalization, - // the followers might have this in the Raft Log and not yet processed it. - switch (haltingPoint) { - case BEFORE_PRE_FINALIZE_UPGRADE: - // At least one node (leader) should be in the FINALIZATION_REQUIRED stage. - assertTrue(scms.stream().anyMatch(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_REQUIRED)); - // Pipeline creation should not be frozen at this point, even on leader. - assertTrue(scms.stream().noneMatch(scm -> - scm.getPipelineManager().isPipelineCreationFrozen())); - break; - case AFTER_PRE_FINALIZE_UPGRADE: - // At least one node (leader) should be in the FINALIZATION_STARTED stage. - assertTrue(scms.stream().anyMatch(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_STARTED)); - // Pipeline creation should be frozen on nodes where the finalization checkpoint is FINALIZATION_STARTED, - // this should include the leader SCM. - assertTrue(scms.stream() - .filter(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_STARTED) - .allMatch(scm -> - scm.getPipelineManager().isPipelineCreationFrozen())); - break; - case AFTER_COMPLETE_FINALIZATION: - // At least one node (leader) should be in the MLV_EQUALS_SLV stage. - assertTrue(scms.stream().anyMatch(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.MLV_EQUALS_SLV)); - // Pipeline creation should not be frozen on nodes where the finalization checkpoint is MLV_EQUALS_SLV, - // this should include the leader SCM. - assertTrue(scms.stream() - .filter(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.MLV_EQUALS_SLV) - .noneMatch(scm -> - scm.getPipelineManager().isPipelineCreationFrozen())); - break; - case AFTER_POST_FINALIZE_UPGRADE: - // At least one node (leader) should be in the FINALIZATION_COMPLETE stage. - assertTrue(scms.stream().anyMatch(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_COMPLETE)); - // Pipeline creation should not be frozen on nodes where the finalization checkpoint is FINALIZATION_COMPLETE, - // this should include the leader SCM. - assertTrue(scms.stream() - .filter(scm -> - scm.getScmContext().getFinalizationCheckpoint() == FinalizationCheckpoint.FINALIZATION_COMPLETE) - .noneMatch(scm -> - scm.getPipelineManager().isPipelineCreationFrozen())); - break; - default: - fail("Unknown halting point in test: " + haltingPoint); - } + assertThat(logCapture.getOutput()).contains("New snapshot received with higher apparent version " + + HDDSVersion.SOFTWARE_VERSION + ". Attempting to finalize to that version."); } } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java index 482662c9df8,3c1d3ed6faf..0c3bc3b37bc --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java @@@ -41,9 -45,11 +41,9 @@@ import org.apache.hadoop.hdds.conf.Ozon import org.apache.hadoop.hdds.scm.block.BlockManager; import org.apache.hadoop.hdds.scm.container.placement.metrics.SCMPerformanceMetrics; import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol; -import org.apache.hadoop.hdds.scm.server.SCMConfigurator; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; - import org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils; -import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; + import org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.UniformDatanodesFactory; import org.apache.hadoop.ozone.client.OzoneBucket; @@@ -136,8 -151,18 +136,8 @@@ public class TestBlockDeletionService GenericTestUtils.waitFor(() -> metrics.getDeleteKeyFailedBlocks() - initialFailedBlocks == 0, 50, 1000); // UPGRADE SCM (if specified) - // Step 5: wait for finalizing upgrade - Future<?> finalizationFuture = Executors.newSingleThreadExecutor().submit(() -> { - try { - scmClient.finalizeScmUpgrade(CLIENT_ID); - } catch (IOException ex) { - fail("finalization client failed", ex); - } - }); - finalizationFuture.get(); - HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - assertEquals(STORAGE_SPACE_DISTRIBUTION.ordinal(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + scmClient.finalizeUpgrade(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient); ++ HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient); // POST-UPGRADE //Step 6: Repeat the same steps in pre-upgrade diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java index f6e0baec5e5,44131d81c88..aea70b1d253 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotTests.java @@@ -51,8 -51,11 +51,9 @@@ import static org.apache.hadoop.ozone.s import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.CANCELLED; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.DONE; import static org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse.JobStatus.IN_PROGRESS; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isDone; -import static org.apache.hadoop.ozone.upgrade.UpgradeFinalization.isStarting; import static org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer.COLUMN_FAMILIES_TO_TRACK_IN_DAG; import static org.apache.ozone.test.LambdaTestUtils.await; + import static org.apache.ozone.test.OzoneTestBase.uniqueObjectName; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java index 5bfbe349e83,884c5fa310b..f913b7a79ad --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java @@@ -111,12 -111,12 +111,12 @@@ public abstract class OMClientRequest i */ public OMRequest preExecute(OzoneManager ozoneManager) throws IOException { - LayoutVersion layoutVersion = LayoutVersion.newBuilder() - .setVersion(ozoneManager.getVersionManager().getMetadataLayoutVersion()) + LayoutVersion apparentVersion = LayoutVersion.newBuilder() + .setVersion(ozoneManager.getVersionManager().getApparentVersion().serialize()) .build(); omRequest = getOmRequest().toBuilder() - .setUserInfo(getUserIfNotExists(ozoneManager)) - .setLayoutVersion(layoutVersion).build(); + .setLayoutVersion(apparentVersion) - .setUserInfo(getUserInfo()).build(); ++ .setUserInfo(getUserIfNotExists(ozoneManager)).build(); return omRequest; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
