This is an automated email from the ASF dual-hosted git repository. adoroszlai pushed a commit to branch HDDS-10656-atomic-key-overwrite in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 1fe90111a3fbdbdf0101b200f14dc7227c982fcc Merge: 99be3179a4 9f1f7ed238 Author: Doroszlai, Attila <[email protected]> AuthorDate: Fri Jun 21 10:30:37 2024 +0200 Merge remote-tracking branch 'origin/master' into HDDS-10656-atomic-key-overwrite .github/workflows/build-ratis.yml | 137 +++ .github/workflows/ci-with-ratis.yml | 52 + .github/workflows/ci.yml | 66 +- .github/workflows/intermittent-test-check.yml | 62 +- .github/workflows/populate-cache.yml | 24 +- .github/workflows/repeat-acceptance.yml | 2 +- .gitignore | 1 + .mvn/{gradle-enterprise.xml => develocity.xml} | 15 +- .mvn/extensions.xml | 6 +- dev-support/ci/download-nodejs.sh | 63 ++ .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 6 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 68 +- .../hadoop/hdds/scm/XceiverClientMetrics.java | 59 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 92 +- .../hadoop/hdds/scm/client/ClientTrustManager.java | 2 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 31 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 25 +- .../ozone/client/io/BlockInputStreamFactory.java | 3 +- .../client/io/BlockInputStreamFactoryImpl.java | 3 +- .../hadoop/ozone/client/io/ECBlockInputStream.java | 7 +- .../hdds/scm/storage/DummyBlockInputStream.java | 2 +- .../storage/DummyBlockInputStreamWithRetry.java | 2 +- .../hdds/scm/storage/TestBlockInputStream.java | 2 +- .../storage/TestBlockOutputStreamCorrectness.java | 10 +- .../client/io/TestBlockInputStreamFactoryImpl.java | 11 +- hadoop-hdds/common/pom.xml | 12 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 7 - .../java/org/apache/hadoop/hdds/HddsUtils.java | 6 +- .../org/apache/hadoop/hdds/client/BlockID.java | 57 +- .../hadoop/hdds/client/DecommissionUtils.java | 153 +++ .../org/apache/hadoop/hdds/client/OzoneQuota.java | 124 ++- .../org/apache/hadoop/hdds/client/QuotaList.java | 69 -- .../hadoop/hdds/conf/OzoneConfiguration.java | 9 +- .../hadoop/hdds/freon/FakeClusterTopology.java | 38 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 135 ++- .../hdds/ratis/ContainerCommandRequestMessage.java | 4 + .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 7 + .../hadoop/hdds/ratis/conf/RatisClientConfig.java | 22 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 4 +- .../java/org/apache/hadoop/hdds/scm/ScmInfo.java | 11 +- .../apache/hadoop/hdds/scm/XceiverClientReply.java | 3 +- .../scm/container/common/helpers/ExcludeList.java | 19 +- .../hadoop/hdds/scm/exceptions/SCMException.java | 6 + .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +- .../apache/hadoop/hdds/scm/net/NetConstants.java | 3 + .../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 55 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 29 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 53 +- .../hadoop/hdds/security/SecurityConfig.java | 51 +- .../x509/certificate/utils/CertificateCodec.java | 107 +- .../certificate/utils/SelfSignedCertificate.java | 24 +- .../hadoop/hdds/security/x509/crl/CRLStatus.java | 87 -- .../hdds/security/x509/crl/package-info.java | 24 - .../org/apache/hadoop/hdds/server/JsonUtils.java | 34 +- .../apache/hadoop/hdds/utils/BatchOperation.java | 88 -- .../org/apache/hadoop/ozone/ClientVersion.java | 4 + .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 2 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 8 +- .../apache/hadoop/ozone/OzoneManagerVersion.java | 4 +- .../org/apache/hadoop/ozone/OzoneSecurityUtil.java | 5 +- .../apache/hadoop/ozone/common/ChecksumData.java | 5 +- .../ozone/common/statemachine/StateMachine.java | 10 +- .../ozone/container/common/helpers/ChunkInfo.java | 13 +- .../java/org/apache/hadoop/ozone/lease/Lease.java | 26 +- .../hadoop/ozone/lease/LeaseCallbackExecutor.java | 14 +- .../apache/hadoop/ozone/lease/LeaseManager.java | 5 +- .../hadoop/ozone/util/ShutdownHookManager.java | 38 +- .../org/apache/hadoop/util/PerformanceMetrics.java | 23 +- .../hadoop/util/PerformanceMetricsInitializer.java | 24 +- .../apache/hadoop/util/StringWithByteString.java | 54 + .../common/src/main/resources/ozone-default.xml | 15 +- .../org/apache/hadoop/hdds/JsonTestUtils.java} | 69 +- .../apache/hadoop/hdds/client/TestOzoneQuota.java} | 24 +- .../ratis/TestContainerCommandRequestMessage.java | 3 + .../hdds/security/x509/CertificateTestUtils.java | 1 + .../ozone/container/ContainerTestHelper.java | 22 +- .../hdds/datanode/metadata/CRLDBDefinition.java | 100 -- .../hdds/datanode/metadata/DatanodeCRLStore.java | 84 -- .../datanode/metadata/DatanodeCRLStoreImpl.java | 128 --- .../apache/hadoop/ozone/HddsDatanodeService.java | 21 +- .../container/common/helpers/ContainerMetrics.java | 61 +- .../container/common/helpers/DatanodeIdYaml.java | 6 +- .../container/common/impl/HddsDispatcher.java | 3 +- .../common/report/CRLStatusReportPublisher.java | 82 -- .../common/report/ReportPublisherFactory.java | 2 - .../common/statemachine/DatanodeStateMachine.java | 12 +- .../common/statemachine/StateContext.java | 13 - .../ClosePipelineCommandHandler.java | 32 +- .../common/transport/server/ratis/CSMMetrics.java | 16 +- .../server/ratis/ContainerStateMachine.java | 45 +- .../ozone/container/common/volume/VolumeUsage.java | 39 +- .../KeyValueContainerMetadataInspector.java | 178 ++-- .../ozone/container/keyvalue/KeyValueHandler.java | 23 +- .../container/keyvalue/helpers/BlockUtils.java | 23 +- .../container/keyvalue/helpers/ChunkUtils.java | 4 +- .../container/keyvalue/impl/BlockManagerImpl.java | 20 +- .../keyvalue/impl/KeyValueStreamDataChannel.java | 7 +- .../keyvalue/impl/StreamDataChannelBase.java | 3 + .../keyvalue/interfaces/BlockManager.java | 4 +- .../keyvalue/interfaces/ChunkManager.java | 2 +- .../metadata/TestDatanodeCRLStoreImpl.java | 117 -- .../hdds/datanode/metadata/package-info.java | 22 - .../hadoop/ozone/TestHddsDatanodeService.java | 24 +- .../hadoop/ozone/TestHddsSecureDatanodeInit.java | 59 +- .../container/common/helpers/TestBlockData.java | 2 +- .../container/common/impl/TestHddsDispatcher.java | 4 +- .../common/report/TestReportPublisher.java | 74 -- .../common/report/TestReportPublisherFactory.java | 12 - .../common/statemachine/TestStateContext.java | 11 +- .../volume/TestCapacityVolumeChoosingPolicy.java | 4 + .../common/volume/TestReservedVolumeSpace.java | 60 +- .../volume/TestRoundRobinVolumeChoosingPolicy.java | 5 + .../TestKeyValueContainerMetadataInspector.java | 94 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 59 ++ .../impl/TestKeyValueStreamDataChannel.java | 4 +- hadoop-hdds/crypto-api/pom.xml | 37 + hadoop-hdds/crypto-default/pom.xml | 37 + hadoop-hdds/docs/content/concept/Recon.zh.md | 2 +- .../content/concept/StorageContainerManager.md | 2 +- hadoop-hdds/docs/content/start/OnPrem.md | 2 +- hadoop-hdds/docs/content/start/OnPrem.zh.md | 6 +- hadoop-hdds/docs/dev-support/bin/generate-site.sh | 2 +- .../hdds/conf/DatanodeRatisServerConfig.java | 8 +- .../apache/hadoop/hdds/conf/HddsConfServlet.java | 10 +- .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 31 +- .../SCMSecurityProtocolClientSideTranslatorPB.java | 61 +- .../DeletedBlocksTransactionInfoWrapper.java | 9 +- .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 46 - .../scm/update/client/CRLClientUpdateHandler.java | 202 ---- .../hadoop/hdds/scm/update/client/CRLStore.java | 34 - .../hdds/scm/update/client/ClientCRLStore.java | 98 -- .../scm/update/client/ClientUpdateHandler.java | 29 - .../client/SCMUpdateClientConfiguration.java | 53 - .../update/client/SCMUpdateServiceGrpcClient.java | 219 ---- .../scm/update/client/UpdateServiceConfig.java | 43 - .../hdds/scm/update/client/package-info.java | 22 - .../hdds/scm/update/server/CRLClientInfo.java | 56 - .../scm/update/server/SCMUpdateClientInfo.java | 67 -- .../hdds/scm/update/server/package-info.java | 22 - .../security/token/CompositeTokenVerifier.java | 4 +- .../hdds/security/token/NoopTokenVerifier.java | 4 +- .../security/token/ShortLivedTokenVerifier.java | 8 +- .../hadoop/hdds/security/token/TokenVerifier.java | 9 +- .../x509/certificate/authority/BaseApprover.java | 18 +- .../x509/certificate/authority/CRLApprover.java | 43 - .../certificate/authority/CertificateApprover.java | 24 +- .../certificate/authority/CertificateServer.java | 46 +- .../certificate/authority/CertificateStore.java | 93 +- .../certificate/authority/DefaultApprover.java | 26 +- .../certificate/authority/DefaultCAServer.java | 164 +-- .../certificate/authority/DefaultCRLApprover.java | 58 - .../client/DefaultCertificateClient.java | 2 +- .../certificate/client/SCMCertificateClient.java | 19 +- .../hadoop/hdds/security/x509/crl/CRLCodec.java | 220 ---- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 226 ---- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 4 + .../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 2 +- .../hadoop/hdds/conf/TestHddsConfServlet.java | 5 +- .../token/TestOzoneBlockTokenSecretManager.java | 16 +- .../hdds/security/token/TokenVerifierTests.java | 14 +- .../x509/certificate/authority/MockApprover.java | 58 - .../x509/certificate/authority/MockCAStore.java | 56 +- .../certificate/authority/TestDefaultCAServer.java | 135 +-- .../certificate/authority/TestDefaultProfile.java | 33 +- .../client/CertificateClientTestImpl.java | 64 +- .../client/TestDefaultCertificateClient.java | 12 +- .../client/TestDnCertificateClientInit.java | 7 +- .../client/TestRootCaRotationPoller.java | 15 +- .../x509/certificate/utils/TestCRLCodec.java | 288 ----- .../certificate/utils/TestCertificateCodec.java | 174 +-- .../certificate/utils/TestRootCertificate.java | 69 +- .../apache/hadoop/hdds/server/TestJsonUtils.java | 5 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 39 + .../src/main/proto/DatanodeClientProtocol.proto | 1 + .../src/main/proto/ScmServerProtocol.proto | 3 + .../src/main/proto/ScmServerSecurityProtocol.proto | 20 + .../utils/db/managed/ManagedSstFileReader.java | 21 +- hadoop-hdds/pom.xml | 2 + .../src/main/patches/rocks-native.patch | 5 +- hadoop-hdds/rocksdb-checkpoint-differ/pom.xml | 37 - .../ozone/rocksdb/util/SstFileSetReader.java | 33 +- .../ozone/rocksdiff/RocksDBCheckpointDiffer.java | 104 +- .../org/apache/ozone/rocksdiff/RocksDiffUtils.java | 9 +- .../rocksdiff/TestRocksDBCheckpointDiffer.java | 107 +- .../ContainerBalancerSelectionCriteria.java | 18 +- .../container/balancer/ContainerBalancerTask.java | 104 +- .../container/balancer/ContainerMoveSelection.java | 22 + .../algorithms/SCMContainerPlacementCapacity.java | 6 + .../algorithms/SCMContainerPlacementRandom.java | 6 + .../placement/metrics/SCMPerformanceMetrics.java | 94 ++ .../container/replication/ReplicationManager.java | 6 +- .../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 - .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 19 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 9 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 4 +- .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 11 - .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 45 - .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 77 -- .../hdds/scm/node/NodeDecommissionManager.java | 28 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 7 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 13 +- .../hadoop/hdds/scm/node/states/NodeStateMap.java | 71 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 31 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 60 +- .../hdds/scm/security/CRLStatusReportHandler.java | 87 -- .../hdds/scm/security/RootCARotationManager.java | 36 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 8 + .../hadoop/hdds/scm/server/SCMCertStore.java | 205 +--- .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 14 - .../hdds/scm/server/SCMSecurityProtocolServer.java | 71 +- .../hdds/scm/server/StorageContainerManager.java | 54 +- .../hadoop/hdds/scm/update/server/SCMCRLStore.java | 51 - .../scm/update/server/SCMCRLUpdateHandler.java | 142 --- .../scm/update/server/SCMUpdateClientManager.java | 149 --- .../hdds/scm/update/server/SCMUpdateHandler.java | 53 - .../update/server/SCMUpdateServiceGrpcServer.java | 100 -- .../scm/update/server/SCMUpdateServiceImpl.java | 119 --- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 16 - .../hdds/scm/container/balancer/MockedSCM.java | 35 +- .../TestContainerBalancerDatanodeNodeLimit.java | 165 ++- .../balancer/TestContainerBalancerTask.java | 167 +-- .../scm/container/balancer/TestableCluster.java | 8 +- .../TestSCMContainerPlacementCapacity.java | 3 +- .../TestSCMContainerPlacementRandom.java | 6 +- .../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 63 +- .../hdds/scm/ha/io/TestX509CertificateCodec.java | 8 + .../scm/metadata/TestX509CertificateCodec.java | 8 + .../scm/security/TestCRLStatusReportHandler.java | 137 --- .../scm/security/TestRootCARotationManager.java | 7 +- .../hadoop/hdds/scm/server/TestSCMCertStore.java | 198 +--- .../hdds/scm/update/server/MockCRLStore.java | 140 --- .../server/TestSCMUpdateServiceGrpcServer.java | 302 ------ .../ozone/container/common/TestEndPoint.java | 45 +- .../placement/TestContainerPlacement.java | 6 +- .../hadoop/hdds/scm/cli/cert/ListSubcommand.java | 15 +- .../cli/datanode/DecommissionStatusSubCommand.java | 50 +- .../scm/cli/pipeline/ClosePipelineSubcommand.java | 50 +- .../scm/cli/pipeline/FilterPipelineOptions.java | 85 ++ .../scm/cli/pipeline/ListPipelinesSubcommand.java | 77 +- .../apache/hadoop/ozone/client/ObjectStore.java | 92 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 96 +- .../org/apache/hadoop/ozone/client/OzoneKey.java | 25 +- .../hadoop/ozone/client/OzoneKeyDetails.java | 8 +- .../client/checksum/ECBlockChecksumComputer.java | 6 +- .../client/checksum/ECFileChecksumHelper.java | 7 +- .../checksum/ReplicatedFileChecksumHelper.java | 8 +- .../ozone/client/io/ECBlockOutputStreamEntry.java | 4 +- .../hadoop/ozone/client/io/KeyInputStream.java | 8 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 2 +- .../ozone/client/protocol/ClientProtocol.java | 74 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 177 +++- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 37 +- .../ozone/om/exceptions/OMNotLeaderException.java | 7 + .../apache/hadoop/ozone/om/helpers/ErrorInfo.java | 33 +- .../apache/hadoop/ozone/om/helpers/OmKeyArgs.java | 20 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +- .../apache/hadoop/ozone/om/helpers/WithTags.java | 20 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 26 +- .../ozone/om/protocolPB/OmTransportFactory.java | 4 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 50 +- .../ozone/snapshot/CancelSnapshotDiffResponse.java | 2 +- .../ozone/snapshot/ListSnapshotResponse.java | 45 +- .../ozone/snapshot/SnapshotDiffReportOzone.java | 24 +- hadoop-ozone/dev-support/checks/acceptance.sh | 4 +- hadoop-ozone/dev-support/checks/blockade.sh | 2 +- hadoop-ozone/dev-support/checks/coverage.sh | 4 +- hadoop-ozone/dev-support/checks/junit.sh | 6 +- hadoop-ozone/dev-support/checks/kubernetes.sh | 2 +- hadoop-ozone/dev-support/checks/native.sh | 2 +- hadoop-ozone/dev-support/checks/unit.sh | 1 - .../dist/dev-support/bin/dist-layout-stitching | 3 + .../dist/src/main/compose/common/s3a-test.sh | 3 +- .../ps.sh => compose_v2_compatibility.sh} | 9 +- hadoop-ozone/dist/src/main/compose/ozone/run.sh | 2 + .../dist/src/main/compose/ozonescripts/README.md | 2 +- .../dist/src/main/compose/ozonescripts/ps.sh | 5 +- .../dist/src/main/compose/ozonescripts/start.sh | 3 + .../dist/src/main/compose/ozonescripts/stop.sh | 3 + .../src/main/compose/ozonesecure/docker-config | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 29 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 2 +- .../dist/src/main/keytabs/update-keytabs.sh | 2 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 3 + .../dist/src/main/license/update-jar-report.sh | 2 +- .../dist/src/main/smoketest/admincli/cert.robot | 4 +- .../dist/src/main/smoketest/env-compose.robot | 32 - .../main/smoketest/httpfs/operations_tests.robot | 10 - .../src/main/smoketest/s3/MultipartUpload.robot | 25 +- .../dist/src/main/smoketest/s3/objectcopy.robot | 21 +- .../dist/src/main/smoketest/s3/objectputget.robot | 32 +- .../src/main/smoketest/security/admin-cert.robot | 5 +- .../src/main/smoketest/snapshot/snapshot-sh.robot | 10 + .../dist/src/shell/ozone/ozone-functions.sh | 5 +- .../apache/ozone/fs/http/server/HttpFSServer.java | 12 +- .../dev-support/findbugsExcludeFile.xml | 14 +- .../fs/ozone/AbstractOzoneFileSystemTest.java | 37 + .../ozone/AbstractOzoneFileSystemTestWithFSO.java | 2 +- .../ozone/AbstractRootedOzoneFileSystemTest.java | 6 + .../hadoop/hdds/scm/TestContainerSmallFile.java | 2 +- .../TestSCMContainerPlacementPolicyMetrics.java | 163 --- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 2 - .../hadoop/hdds/scm/TestXceiverClientGrpc.java | 22 +- .../TestContainerStateManagerIntegration.java | 5 +- .../hdds/scm/pipeline/TestPipelineClose.java | 14 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 10 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 - .../hadoop/ozone/TestContainerOperations.java | 56 + .../TestContainerStateMachineIdempotency.java | 116 -- .../apache/hadoop/ozone/TestDelegationToken.java | 2 - .../hadoop/ozone/TestMultipartObjectGet.java | 10 + .../hadoop/ozone/TestOzoneConfigurationFields.java | 1 - .../hadoop/ozone/TestSecureOzoneCluster.java | 157 ++- ...lientAbstract.java => OzoneRpcClientTests.java} | 514 +++++++-- .../ozone/client/rpc/TestBlockOutputStream.java | 75 +- ...ayCommitInRatis.java => TestCommitInRatis.java} | 30 +- .../client/rpc/TestFailureHandlingByClient.java | 19 +- .../client/rpc/TestOzoneAtRestEncryption.java | 8 - .../ozone/client/rpc/TestOzoneRpcClient.java | 16 +- .../client/rpc/TestOzoneRpcClientWithRatis.java | 348 +----- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 125 +-- .../ozone/client/rpc/TestWatchForCommit.java | 198 ++-- .../ozone/container/TestContainerReplication.java | 209 ++++ .../ozoneimpl/TestOzoneContainerWithTLS.java | 7 + .../org/apache/hadoop/ozone/debug/TestLDBCli.java | 24 + .../hadoop/ozone/freon/TestDNRPCLoadGenerator.java | 35 +- .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 9 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 28 +- .../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +- .../ozone/recon/TestReconContainerEndpoint.java | 222 ++++ .../hadoop/ozone/recon/TestReconScmHASnapshot.java | 2 +- ...napshot.java => TestReconScmNonHASnapshot.java} | 11 +- .../ozone/recon/TestReconWithOzoneManager.java | 10 +- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 20 +- .../shell/TestOzoneContainerUpgradeShell.java | 2 + .../hadoop/ozone/shell/TestOzoneRepairShell.java | 133 +++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 26 +- .../hadoop/ozone/shell/TestReconfigShell.java | 3 + .../src/main/proto/OmClientProtocol.proto | 17 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 7 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 54 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 40 +- .../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 12 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 16 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 + .../ozone/om/ratis/OzoneManagerStateMachine.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 9 + .../ozone/om/request/key/OMKeysDeleteRequest.java | 17 +- .../om/request/key/OmKeysDeleteRequestWithFSO.java | 12 +- .../S3ExpiredMultipartUploadsAbortRequest.java | 3 + .../S3InitiateMultipartUploadRequest.java | 3 + .../S3InitiateMultipartUploadRequestWithFSO.java | 3 + .../multipart/S3MultipartUploadAbortRequest.java | 2 + .../S3MultipartUploadCommitPartRequest.java | 1 + .../S3MultipartUploadCompleteRequest.java | 7 +- .../protocolPB/OzoneManagerRequestHandler.java | 18 +- .../OzoneDelegationTokenSecretManager.java | 2 + .../apache/hadoop/ozone/om/TestChunkStreams.java | 5 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 61 +- .../om/ratis/TestOzoneManagerStateMachine.java | 20 + .../ozone/om/request/OMRequestTestUtils.java | 7 +- .../om/request/key/TestOMKeyCreateRequest.java | 62 +- .../om/request/key/TestOMKeysDeleteRequest.java | 7 + .../TestS3InitiateMultipartUploadRequest.java | 9 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 9 +- .../s3/multipart/TestS3MultipartRequest.java | 43 +- .../TestS3MultipartUploadCompleteRequest.java | 20 +- .../security/TestOmCertificateClientInit.java | 4 +- .../TestOzoneDelegationTokenSecretManager.java | 12 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 24 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 6 + .../ozone/BasicRootedOzoneClientAdapterImpl.java | 7 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 6 + .../apache/hadoop/fs/ozone/OzoneClientUtils.java | 9 +- hadoop-ozone/recon/pom.xml | 2 +- .../apache/hadoop/ozone/recon/ReconConstants.java | 1 + .../apache/hadoop/ozone/recon/ReconContext.java | 149 +++ .../hadoop/ozone/recon/ReconControllerModule.java | 1 + .../hadoop/ozone/recon/ReconResponseUtils.java | 84 ++ .../org/apache/hadoop/ozone/recon/ReconServer.java | 16 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 1 + .../org/apache/hadoop/ozone/recon/ReconUtils.java | 221 +++- .../ozone/recon/api/ClusterStateEndpoint.java | 13 +- .../hadoop/ozone/recon/api/ContainerEndpoint.java | 51 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 25 +- .../hadoop/ozone/recon/api/NodeEndpoint.java | 137 ++- .../ozone/recon/api/OMDBInsightEndpoint.java | 655 +++++++++++- .../recon/api/handlers/DirectoryEntityHandler.java | 1 - .../ozone/recon/api/handlers/EntityHandler.java | 2 +- .../ozone/recon/api/handlers/FSOBucketHandler.java | 1 - .../recon/api/types/ClusterStateResponse.java | 29 + .../ozone/recon/api/types/DatanodeMetrics.java | 81 ++ .../api/types/DecommissionStatusInfoResponse.java | 73 ++ .../ozone/recon/api/types/KeyEntityInfo.java | 42 +- .../hadoop/ozone/recon/api/types/KeyMetadata.java | 11 + .../hadoop/ozone/recon/api/types/KeysResponse.java | 9 +- .../ozone/recon/api/types/ListKeysResponse.java | 114 ++ .../hadoop/ozone/recon/api/types/NSSummary.java | 15 +- .../hadoop/ozone/recon/api/types/ParamInfo.java | 133 +++ .../hadoop/ozone/recon/codec/NSSummaryCodec.java | 17 +- .../ozone/recon/fsck/ContainerHealthTask.java | 14 +- .../hadoop/ozone/recon/scm/PipelineSyncTask.java | 2 +- .../ozone/recon/scm/ReconContainerManager.java | 8 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 66 +- .../ozone/recon/scm/ReconPipelineManager.java | 79 +- .../recon/scm/ReconPipelineReportHandler.java | 7 +- .../scm/ReconStorageContainerManagerFacade.java | 29 +- .../recon/spi/ReconNamespaceSummaryManager.java | 3 + .../spi/impl/ReconNamespaceSummaryManagerImpl.java | 12 +- .../impl/StorageContainerServiceProviderImpl.java | 110 +- .../ozone/recon/tasks/ContainerKeyMapperTask.java | 8 +- .../ozone/recon/tasks/ContainerSizeCountTask.java | 6 +- .../ozone/recon/tasks/FileSizeCountTask.java | 6 +- .../hadoop/ozone/recon/tasks/NSSummaryTask.java | 2 +- .../recon/tasks/NSSummaryTaskDbEventHandler.java | 2 + .../ozone/recon/tasks/NSSummaryTaskWithFSO.java | 4 +- .../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 16 +- .../ozone/recon/tasks/NSSummaryTaskWithOBS.java | 6 +- .../ozone/recon/tasks/OmTableInsightTask.java | 4 +- .../webapps/recon/ozone-recon-web/api/db.json | 296 +++++- .../webapps/recon/ozone-recon-web/api/routes.json | 31 +- .../src/components/overviewCard/overviewCard.tsx | 5 + .../src/components/rightDrawer/rightDrawer.tsx | 4 +- .../src/utils/axiosRequestHelper.tsx | 15 + .../src/views/datanodes/datanodes.less | 4 + .../src/views/datanodes/datanodes.tsx | 84 +- .../src/views/diskUsage/diskUsage.tsx | 41 +- .../ozone-recon-web/src/views/heatMap/heatmap.tsx | 65 +- .../src/views/overview/overview.tsx | 23 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 57 +- ...teCounts.java => TestClusterStateEndpoint.java} | 28 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 164 ++- .../recon/api/TestNSSummaryEndpointWithFSO.java | 166 ++- .../recon/api/TestNSSummaryEndpointWithLegacy.java | 48 +- .../api/TestNSSummaryEndpointWithOBSAndLegacy.java | 76 +- .../ozone/recon/api/TestOmDBInsightEndPoint.java | 1116 +++++++++++++++++++- .../ozone/recon/heatmap/TestHeatMapInfo.java | 440 ++++---- .../ozone/recon/scm/TestReconNodeManager.java | 50 +- .../ozone/recon/scm/TestReconPipelineManager.java | 36 +- .../impl/TestReconNamespaceSummaryManagerImpl.java | 6 +- .../recon/tasks/TestNSSummaryTaskWithFSO.java | 54 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 61 +- ...CompleteMultipartUploadRequestUnmarshaller.java | 6 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 84 ++ .../endpoint/MultiDeleteRequestUnmarshaller.java | 6 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 93 +- .../ozone/s3/endpoint/ObjectEndpointStreaming.java | 5 +- .../endpoint/PutBucketAclRequestUnmarshaller.java | 6 +- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 + .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 14 + .../hadoop/ozone/client/ClientProtocolStub.java | 38 +- .../hadoop/ozone/client/OzoneBucketStub.java | 46 +- .../hadoop/ozone/client/OzoneOutputStreamStub.java | 18 - ...CompleteMultipartUploadRequestUnmarshaller.java | 44 + .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 75 +- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 229 +++- .../ozone/s3/endpoint/TestPermissionCheck.java | 13 +- .../ozone/admin/nssummary/DiskUsageSubCommand.java | 4 - .../ozone/admin/nssummary/NSSummaryAdmin.java | 6 + .../ozone/admin/om/DecommissionOMSubcommand.java | 4 +- .../scm/ResetDeletedBlockRetryCountSubcommand.java | 15 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 26 +- .../apache/hadoop/ozone/debug/RocksDBUtils.java | 31 + .../hadoop/ozone/freon/BaseFreonGenerator.java | 15 +- .../hadoop/ozone/freon/DNRPCLoadGenerator.java | 31 +- .../hadoop/ozone/freon/OmMetadataGenerator.java | 4 + .../freon/OzoneClientKeyReadWriteListOps.java | 5 + .../GenerateOzoneRequiredConfigurations.java | 3 +- .../apache/hadoop/ozone/repair/OzoneRepair.java | 30 + .../hadoop/ozone/repair/TransactionInfoRepair.java | 135 +++ .../hadoop/ozone/repair/om/SnapshotRepair.java | 34 +- .../apache/hadoop/ozone/shell/OzoneAddress.java | 37 +- .../ozone/shell/snapshot/ListSnapshotHandler.java | 2 +- .../ozone/shell/snapshot/SnapshotDiffHandler.java | 78 +- .../hadoop/ozone/repair/TestOzoneRepair.java | 88 ++ .../ozone/repair/TestTransactionInfoRepair.java | 146 +++ .../shell/TestOzoneAddressClientCreation.java | 30 + .../org.mockito.plugins.MockMaker} | 3 +- pom.xml | 121 ++- 479 files changed, 11745 insertions(+), 9772 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java index f5031a1265,c55945d537..c53aca2f15 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneManagerVersion.java @@@ -40,8 -40,8 +40,10 @@@ public enum OzoneManagerVersion impleme LIGHTWEIGHT_LIST_KEYS(4, "OzoneManager version that supports lightweight" + " listKeys API."), - ATOMIC_REWRITE_KEY(5, "OzoneManager version that supports rewriting key as atomic operation"), + OBJECT_TAG(5, "OzoneManager version that supports object tags"), + ++ ATOMIC_REWRITE_KEY(6, "OzoneManager version that supports rewriting key as atomic operation"), + FUTURE_VERSION(-1, "Used internally in the client when the server side is " + " newer and an unknown server version has arrived to the client."); diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 012f029f51,575701be80..216b51b8e8 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@@ -466,32 -467,29 +467,51 @@@ public class OzoneBucket extends WithMe ReplicationConfig replicationConfig, Map<String, String> keyMetadata) throws IOException { + return this.createKey(key, size, replicationConfig, keyMetadata, Collections.emptyMap()); + } + + /** + * Creates a new key in the bucket. + * + * @param key Name of the key to be created. + * @param size Size of the data the key will point to. + * @param replicationConfig Replication configuration. + * @param keyMetadata Custom key metadata. + * @param tags Tags used for S3 object tags + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream createKey(String key, long size, + ReplicationConfig replicationConfig, + Map<String, String> keyMetadata, + Map<String, String> tags) + throws IOException { return proxy - .createKey(volumeName, name, key, size, replicationConfig, keyMetadata); + .createKey(volumeName, name, key, size, replicationConfig, keyMetadata, tags); } + /** + * This API allows to atomically update an existing key. The key read before invoking this API + * should remain unchanged for this key to be written. This is controlled by the generation + * field in the existing Key param. If the key is replaced or updated the generation will change. If the + * generation has changed since the existing Key was read, either the initial key create will fail, + * or the key will fail to commit after the data has been written as the checks are carried out + * both at key open and commit time. + * + * @param keyName Existing key to rewrite. This must exist in the bucket. + * @param size The size of the new key + * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * and commit time. + * @param replicationConfig The replication configuration for the key to be rewritten. + * @param metadata custom key value metadata + * @return OzoneOutputStream to which the data has to be written. + * @throws IOException + */ + public OzoneOutputStream rewriteKey(String keyName, long size, long existingKeyGeneration, + ReplicationConfig replicationConfig, Map<String, String> metadata) throws IOException { + return proxy.rewriteKey(volumeName, name, keyName, size, existingKeyGeneration, replicationConfig, metadata); + } + /** * Creates a new key in the bucket, with default replication type RATIS and * with replication factor THREE. diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java index cd2978fce1,168e15d9bd..fe547794e9 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneKeyDetails.java @@@ -59,30 -53,12 +59,30 @@@ public class OzoneKeyDetails extends Oz Map<String, String> metadata, FileEncryptionInfo feInfo, CheckedSupplier<OzoneInputStream, IOException> contentSupplier, - boolean isFile, String owner, Long generation) { - boolean isFile, String owner, Map<String, String> tags) { ++ boolean isFile, String owner, Map<String, String> tags, Long generation) { super(volumeName, bucketName, keyName, size, creationTime, - modificationTime, replicationConfig, metadata, isFile, owner); + modificationTime, replicationConfig, metadata, isFile, owner, tags); this.ozoneKeyLocations = ozoneKeyLocations; this.feInfo = feInfo; this.contentSupplier = contentSupplier; + this.generation = generation; + } + + /** + * Constructs OzoneKeyDetails from OmKeyInfo. + */ + @SuppressWarnings("parameternumber") + public OzoneKeyDetails(String volumeName, String bucketName, String keyName, + long size, long creationTime, long modificationTime, + List<OzoneKeyLocation> ozoneKeyLocations, + ReplicationConfig replicationConfig, + Map<String, String> metadata, + FileEncryptionInfo feInfo, + CheckedSupplier<OzoneInputStream, IOException> contentSupplier, - boolean isFile, String owner) { ++ boolean isFile, String owner, Map<String, String> tags) { + this(volumeName, bucketName, keyName, size, creationTime, + modificationTime, ozoneKeyLocations, replicationConfig, metadata, feInfo, contentSupplier, - isFile, owner, null); ++ isFile, owner, tags, null); } /** diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 0725b4f253,74344344f8..55985a18d3 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@@ -353,29 -355,22 +355,45 @@@ public interface ClientProtocol Map<String, String> metadata) throws IOException; + /** + * This API allows to atomically update an existing key. The key read before invoking this API + * should remain unchanged for this key to be written. This is controlled by the generation + * field in the existing Key param. If the key is replaced or updated the generation will change. If the + * generation has changed since the existing Key was read, either the initial key create will fail, + * or the key will fail to commit after the data has been written as the checks are carried out + * both at key open and commit time. + * + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Existing key to rewrite. This must exist in the bucket. + * @param size The size of the new key + * @param existingKeyGeneration The generation of the existing key which is checked for changes at key create + * and commit time. + * @param replicationConfig The replication configuration for the key to be rewritten. + * @param metadata custom key value metadata + * @return {@link OzoneOutputStream} + * @throws IOException + */ + OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName, + long size, long existingKeyGeneration, ReplicationConfig replicationConfig, + Map<String, String> metadata) throws IOException; + + /** + * Writes a key in an existing bucket. + * @param volumeName Name of the Volume + * @param bucketName Name of the Bucket + * @param keyName Name of the Key + * @param size Size of the data + * @param metadata Custom key value metadata + * @param tags Tags used for S3 object tags + * @return {@link OzoneOutputStream} + * + */ + OzoneOutputStream createKey(String volumeName, String bucketName, + String keyName, long size, ReplicationConfig replicationConfig, + Map<String, String> metadata, Map<String, String> tags) + throws IOException; + /** * Writes a key in an existing bucket. * @param volumeName Name of the Volume diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index c34987090f,248f21cd70..ac0cf1d09c --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@@ -1387,7 -1387,41 +1387,23 @@@ public class RpcClient implements Clien ReplicationConfig replicationConfig, Map<String, String> metadata) throws IOException { + return createKey(volumeName, bucketName, keyName, size, replicationConfig, + metadata, Collections.emptyMap()); + } + + @Override + public OzoneOutputStream createKey( + String volumeName, String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, + Map<String, String> metadata, Map<String, String> tags) throws IOException { - verifyVolumeName(volumeName); - verifyBucketName(bucketName); - if (checkKeyNameEnabled) { - HddsClientUtils.verifyKeyName(keyName); - } - HddsClientUtils.checkNotNull(keyName); - if (omVersion - .compareTo(OzoneManagerVersion.ERASURE_CODED_STORAGE_SUPPORT) < 0) { - if (replicationConfig != null && - replicationConfig.getReplicationType() - == HddsProtos.ReplicationType.EC) { - throw new IOException("Can not set the replication of the key to" - + " Erasure Coded replication, as OzoneManager does not support" - + " Erasure Coded replication."); - } - } + createKeyPreChecks(volumeName, bucketName, keyName, replicationConfig); + + if (omVersion.compareTo(OzoneManagerVersion.OBJECT_TAG) < 0) { + if (tags != null && !tags.isEmpty()) { + throw new IOException("OzoneManager does not support object tags"); + } + } + - if (replicationConfig != null) { - replicationConfigValidator.validate(replicationConfig); - } String ownerName = getRealUserInfo().getShortUserName(); OmKeyArgs.Builder builder = new OmKeyArgs.Builder() @@@ -1758,8 -1767,8 +1804,10 @@@ keyInfo.getModificationTime(), ozoneKeyLocations, keyInfo.getReplicationConfig(), keyInfo.getMetadata(), keyInfo.getFileEncryptionInfo(), - () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(), keyInfo.getOwnerName(), - keyInfo.getGeneration()); - () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(), - keyInfo.getOwnerName(), keyInfo.getTags()); ++ () -> getInputStreamWithRetryFunction(keyInfo), keyInfo.isFile(), ++ keyInfo.getOwnerName(), keyInfo.getTags(), ++ keyInfo.getGeneration() ++ ); } @Override diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java index 6d0cbc9db0,19d5ab4fa7..ba28b45a0e --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyArgs.java @@@ -53,13 -53,7 +53,14 @@@ public final class OmKeyArgs implement private final boolean recursive; private final boolean headOp; private final boolean forceUpdateContainerCacheFromSCM; + private final Map<String, String> tags; + // expectedDataGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. + private Long expectedDataGeneration = null; private OmKeyArgs(Builder b) { this.volumeName = b.volumeName; @@@ -79,7 -73,7 +80,8 @@@ this.headOp = b.headOp; this.forceUpdateContainerCacheFromSCM = b.forceUpdateContainerCacheFromSCM; this.ownerName = b.ownerName; + this.tags = b.tags; + this.expectedDataGeneration = b.expectedDataGeneration; } public boolean getIsMultipartKey() { @@@ -158,10 -152,10 +160,14 @@@ return forceUpdateContainerCacheFromSCM; } + public Map<String, String> getTags() { + return tags; + } + + public Long getExpectedDataGeneration() { + return expectedDataGeneration; + } + @Override public Map<String, String> toAuditMap() { Map<String, String> auditMap = new LinkedHashMap<>(); @@@ -201,12 -195,8 +207,13 @@@ .setHeadOp(headOp) .setLatestVersionLocation(latestVersionLocation) .setAcls(acls) - .setForceUpdateContainerCacheFromSCM(forceUpdateContainerCacheFromSCM); + .setForceUpdateContainerCacheFromSCM(forceUpdateContainerCacheFromSCM) + .addAllTags(tags); + + if (expectedDataGeneration != null) { + builder.setExpectedDataGeneration(expectedDataGeneration); + } + return builder; } @Nonnull @@@ -248,7 -235,7 +255,8 @@@ private boolean recursive; private boolean headOp; private boolean forceUpdateContainerCacheFromSCM; + private final Map<String, String> tags = new HashMap<>(); + private Long expectedDataGeneration = null; public Builder setVolumeName(String volume) { this.volumeName = volume; diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index a9a0fb82a4,c8e7f8f609..f52a142239 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@@ -102,14 -102,11 +102,19 @@@ public final class OmKeyInfo extends Wi */ private final CopyOnWriteArrayList<OzoneAcl> acls; + /** + * Used for S3 tags. + */ + private Map<String, String> tags; + + // expectedDataGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. + private Long expectedDataGeneration = null; + private OmKeyInfo(Builder b) { super(b); this.volumeName = b.volumeName; @@@ -126,7 -123,7 +131,8 @@@ this.fileName = b.fileName; this.isFile = b.isFile; this.ownerName = b.ownerName; + this.tags = b.tags; + this.expectedDataGeneration = b.expectedDataGeneration; } public String getVolumeName() { @@@ -460,7 -451,7 +476,8 @@@ private FileChecksum fileChecksum; private boolean isFile; + private final Map<String, String> tags = new HashMap<>(); + private Long expectedDataGeneration = null; public Builder() { } @@@ -589,11 -580,16 +606,21 @@@ return this; } + public Builder addTag(String key, String value) { + tags.put(key, value); + return this; + } + + public Builder addAllTags(Map<String, String> keyTags) { + tags.putAll(keyTags); + return this; + } + + public Builder setExpectedDataGeneration(Long existingGeneration) { + this.expectedDataGeneration = existingGeneration; + return this; + } + public OmKeyInfo build() { return new OmKeyInfo(this); } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java index f53fad3bdc,f3c9227126..007591ddc3 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/OzoneRpcClientTests.java @@@ -187,13 -194,10 +199,10 @@@ import org.junit.jupiter.params.provide /** * This is an abstract class to test all the public facing APIs of Ozone - * Client, w/o OM Ratis server. - * {@link TestOzoneRpcClient} tests the Ozone Client by submitting the - * requests directly to OzoneManager. {@link TestOzoneRpcClientWithRatis} - * tests the Ozone Client by submitting requests to OM's Ratis server. + * Client. */ @TestMethodOrder(MethodOrderer.MethodName.class) - public abstract class TestOzoneRpcClientAbstract extends OzoneTestBase { -abstract class OzoneRpcClientTests { ++abstract class OzoneRpcClientTests extends OzoneTestBase { private static MiniOzoneCluster cluster = null; private static OzoneClient ozClient = null; @@@ -4108,38 -3956,19 +4178,32 @@@ assertNotNull(omMultipartUploadCompleteInfo.getHash()); } - private void createTestKey(OzoneBucket bucket, String keyName, - String keyValue) throws IOException { - OzoneOutputStream out = bucket.createKey(keyName, - keyValue.getBytes(UTF_8).length, RATIS, - ONE, new HashMap<>()); - out.write(keyValue.getBytes(UTF_8)); - out.close(); - OzoneKey key = bucket.getKey(keyName); + private OzoneKeyDetails createTestKey(OzoneBucket bucket) throws IOException { + return createTestKey(bucket, getTestName(), UUID.randomUUID().toString()); + } + + private OzoneKeyDetails createTestKey( + OzoneBucket bucket, String keyName, String keyValue + ) throws IOException { + return createTestKey(bucket, keyName, keyValue.getBytes(UTF_8)); + } + + private OzoneKeyDetails createTestKey( + OzoneBucket bucket, String keyName, byte[] bytes + ) throws IOException { + RatisReplicationConfig replication = RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE); + Map<String, String> metadata = singletonMap("key", RandomStringUtils.randomAscii(10)); + try (OzoneOutputStream out = bucket.createKey(keyName, bytes.length, replication, metadata)) { + out.write(bytes); + } + OzoneKeyDetails key = bucket.getKey(keyName); + assertNotNull(key); assertEquals(keyName, key.getName()); + return key; } - private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) - throws Exception { - OMException oe = null; - try { - bucket.getKey(keyName); - } catch (OMException e) { - oe = e; - } + private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) { + OMException oe = assertThrows(OMException.class, () -> bucket.getKey(keyName)); assertEquals(KEY_NOT_FOUND, oe.getResult()); } diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index ccd8682c62,5f5a50d1c8..8640343904 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -1026,13 -1026,9 +1026,17 @@@ message KeyArgs // Force OM to update container cache location from SCL optional bool forceUpdateContainerCacheFromSCM = 20; optional string ownerName = 21; + + // S3 object tags support + repeated hadoop.hdds.KeyValue tags = 22; ++ + // expectedDataGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. - optional uint64 expectedDataGeneration = 22; ++ optional uint64 expectedDataGeneration = 23; } message KeyLocation { @@@ -1116,13 -1112,7 +1120,14 @@@ message KeyInfo optional FileChecksumProto fileChecksum = 18; optional bool isFile = 19; optional string ownerName = 20; + repeated hadoop.hdds.KeyValue tags = 21; + // expectedDataGeneration, when used in key creation indicates that a + // key with the same keyName should exist with the given generation. + // For a key commit to succeed, the original key should still be present with the + // generation unchanged. + // This allows a key to be created an committed atomically if the original has not + // been modified. - optional uint64 expectedDataGeneration = 21; ++ optional uint64 expectedDataGeneration = 22; } message BasicKeyInfo { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 3f9f23d4b8,50bb1053be..a90d7a28ff --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@@ -779,9 -780,12 +779,16 @@@ public abstract class OMKeyRequest exte dbKeyInfo.getMetadata().putAll(KeyValueUtil.getFromProtobuf( keyArgs.getMetadataList())); + // Construct a new tags from KeyArgs + // Clear the old one when the key is overwritten + dbKeyInfo.getTags().clear(); + dbKeyInfo.getTags().putAll(KeyValueUtil.getFromProtobuf( + keyArgs.getTagsList())); + + if (keyArgs.hasExpectedDataGeneration()) { + dbKeyInfo.setExpectedDataGeneration(keyArgs.getExpectedDataGeneration()); + } ++ dbKeyInfo.setFileEncryptionInfo(encInfo); return dbKeyInfo; } diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 1bc141a22e,166edb552c..4bfdd33329 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@@ -61,6 -63,6 +62,8 @@@ import org.apache.hadoop.ozone.protocol .OMRequest; import org.junit.jupiter.params.provider.ValueSource; ++import static java.util.Collections.emptyList; ++import static java.util.Collections.emptyMap; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.RATIS; @@@ -142,8 -143,12 +145,12 @@@ public class TestOMKeyCreateRequest ext when(ozoneManager.getOzoneLockProvider()).thenReturn( new OzoneLockProvider(setKeyPathLock, setFileSystemPaths)); + Map<String, String> tags = new HashMap<>(); + tags.put("tag-key1", "tag-value1"); + tags.put("tag-key2", "tag-value2"); + OMRequest modifiedOmRequest = - doPreExecute(createKeyRequest(false, 0)); - doPreExecute(createKeyRequest(false, 0, Collections.emptyMap(), tags)); ++ doPreExecute(createKeyRequest(false, 0, emptyMap(), tags)); OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(modifiedOmRequest); @@@ -186,9 -191,13 +193,13 @@@ .getOpenKeyTable(omKeyCreateRequest.getBucketLayout()) .get(openKey)); + tags.remove("tag-key1"); + tags.remove("tag-key2"); + tags.put("tag-key3", "tag-value3"); + // Override same key again modifiedOmRequest = - doPreExecute(createKeyRequest(false, 0)); - doPreExecute(createKeyRequest(false, 0, Collections.emptyMap(), tags)); ++ doPreExecute(createKeyRequest(false, 0, emptyMap(), tags)); id = modifiedOmRequest.getCreateKeyRequest().getClientID(); openKey = getOpenKey(id); @@@ -520,7 -534,7 +536,7 @@@ // Create the key request without any initial metadata OMRequest createRequestWithoutMetadata = createKeyRequest(false, 0, keyName, - null, Collections.emptyList()); // Passing 'null' for metadata - null); // Passing 'null' for metadata ++ null, emptyMap(), emptyList()); // Passing 'null' for metadata OMKeyCreateRequest createOmKeyCreateRequest = new OMKeyCreateRequest(createRequestWithoutMetadata, getBucketLayout()); @@@ -543,7 -557,7 +559,7 @@@ // Overwrite the previously created key with new metadata OMRequest overwriteRequestWithMetadata = - createKeyRequest(false, 0, keyName, overwriteMetadata, Collections.emptyList()); - createKeyRequest(false, 0, keyName, overwriteMetadata); ++ createKeyRequest(false, 0, keyName, overwriteMetadata, emptyMap(), emptyList()); OMKeyCreateRequest overwriteOmKeyCreateRequest = new OMKeyCreateRequest(overwriteRequestWithMetadata, getBucketLayout()); @@@ -643,12 -657,23 +659,23 @@@ @SuppressWarnings("parameterNumber") protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber) { - return createKeyRequest(isMultipartKey, partNumber, keyName); - return createKeyRequest(isMultipartKey, partNumber, Collections.emptyMap(), Collections.emptyMap()); ++ return createKeyRequest(isMultipartKey, partNumber, emptyMap(), emptyMap()); + } + + protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, + Map<String, String> metadata, Map<String, String> tags) { - return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, tags); ++ return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, tags, emptyList()); } private OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, String keyName) { - return createKeyRequest(isMultipartKey, partNumber, keyName, null, Collections.emptyList()); - return createKeyRequest(isMultipartKey, partNumber, keyName, null); ++ return createKeyRequest(isMultipartKey, partNumber, keyName, emptyMap()); + } + + protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, + String keyName, + Map<String, String> metadata) { - return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, null); ++ return createKeyRequest(isMultipartKey, partNumber, keyName, metadata, emptyMap(), emptyList()); } /** @@@ -666,7 -693,7 +695,8 @@@ protected OMRequest createKeyRequest(boolean isMultipartKey, int partNumber, String keyName, Map<String, String> metadata, - Map<String, String> tags) { ++ Map<String, String> tags, + List<OzoneAcl> acls) { KeyArgs.Builder keyArgs = KeyArgs.newBuilder() .setVolumeName(volumeName) .setBucketName(bucketName) @@@ -1045,13 -991,9 +1079,13 @@@ assertEquals(NOT_A_FILE, omClientResponse.getOMResponse().getStatus()); } - private void createAndCheck(String keyName) throws Exception { - createAndCheck(keyName, Collections.emptyMap(), Collections.emptyList()); - OMRequest omRequest = createKeyRequest(false, 0, keyName); ++ createAndCheck(keyName, emptyMap(), emptyList()); + } + + private OmKeyInfo createAndCheck(String keyName, Map<String, String> metadata, List<OzoneAcl> acls) + throws Exception { - OMRequest omRequest = createKeyRequest(false, 0, keyName, metadata, acls); ++ OMRequest omRequest = createKeyRequest(false, 0, keyName, metadata, emptyMap(), acls); OMKeyCreateRequest omKeyCreateRequest = getOMKeyCreateRequest(omRequest); diff --cc hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index d7d092c0e5,b75167c89c..bea831a065 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@@ -227,14 -230,16 +230,24 @@@ public class ClientProtocolStub impleme .createKey(keyName, size, replicationConfig, metadata); } + @Override + public OzoneOutputStream createKey(String volumeName, String bucketName, + String keyName, long size, + ReplicationConfig replicationConfig, + Map<String, String> metadata, + Map<String, String> tags) throws IOException { + return getBucket(volumeName, bucketName) + .createKey(keyName, size, replicationConfig, metadata, tags); + } + + @Override + public OzoneOutputStream rewriteKey(String volumeName, String bucketName, String keyName, + long size, long existingKeyGeneration, ReplicationConfig replicationConfig, + Map<String, String> metadata) throws IOException { + return getBucket(volumeName, bucketName) + .rewriteKey(keyName, size, existingKeyGeneration, replicationConfig, metadata); + } + @Override public OzoneInputStream getKey(String volumeName, String bucketName, String keyName) throws IOException { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
