This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-4944 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit f2fdf046662c8975d27009ebacf1bfc4b650f639 Merge: bb67e13 05773af Author: Siyao Meng <[email protected]> AuthorDate: Thu Jan 20 12:01:58 2022 -0800 Merge remote-tracking branch 'asf/master' into HDDS-4944 Conflicts: hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java Includes TestMultiTenantVolume fix by @errose28 Change-Id: I3bb78617d89c133e92575009a9c8640dae0225b0 .gitignore | 1 + CONTRIBUTING.md | 3 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +- hadoop-hdds/common/pom.xml | 15 + .../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 4 + .../hadoop/hdds/conf/OzoneConfiguration.java | 8 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../protocol/StorageContainerLocationProtocol.java | 6 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +- .../common/src/main/resources/ozone-default.xml | 93 ++- .../hadoop/hdds/conf/TestOzoneConfiguration.java | 28 + .../hadoop/hdds/tracing/TestStringCodec.java | 18 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 6 +- .../ozone/container/common/interfaces/Handler.java | 2 +- .../common/statemachine/DatanodeConfiguration.java | 32 - .../common/statemachine/DatanodeStateMachine.java | 7 +- .../CloseContainerCommandHandler.java | 17 +- .../common/transport/server/XceiverServerGrpc.java | 2 +- .../transport/server/ratis/XceiverServerRatis.java | 2 +- .../ozoneimpl/ContainerScrubberConfiguration.java | 65 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 11 + .../container/replication/ContainerDownloader.java | 3 +- .../replication/DownloadAndImportReplicator.java | 19 +- .../container/replication/ReplicationServer.java | 56 +- .../replication/ReplicationSupervisor.java | 8 + .../replication/SimpleContainerDownloader.java | 39 +- .../statemachine/TestDatanodeConfiguration.java | 11 - .../TestCloseContainerCommandHandler.java | 4 +- .../TestContainerScrubberConfiguration.java | 97 +++ .../replication/TestReplicationConfig.java | 75 ++ .../replication/TestReplicationSupervisor.java | 4 +- .../replication/TestSimpleContainerDownloader.java | 13 +- hadoop-hdds/docs/config.yaml | 5 + hadoop-hdds/docs/content/security/SecureOzone.md | 10 +- .../docs/content/security/SecureOzone.zh.md | 34 +- .../docs/content/security/SecuringDatanodes.md | 2 +- .../docs/content/security/SecuringDatanodes.zh.md | 2 +- hadoop-hdds/docs/content/start/OnPrem.md | 1 + hadoop-hdds/docs/content/start/OnPrem.zh.md | 2 + hadoop-hdds/docs/content/tools/TestTools.md | 129 +-- hadoop-hdds/docs/content/tools/TestTools.zh.md | 129 +-- hadoop-hdds/docs/content/tools/_index.md | 1 - hadoop-hdds/docs/content/tools/_index.zh.md | 1 - .../java/org/apache/hadoop/hdds/ExitManager.java | 4 + ...inerLocationProtocolClientSideTranslatorPB.java | 48 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +- .../org/apache/hadoop/hdds/utils/db/DBStore.java | 10 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 11 + .../certificate/authority/TestDefaultCAServer.java | 6 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 24 + .../src/main/proto/ScmAdminProtocol.proto | 17 +- hadoop-hdds/server-scm/pom.xml | 10 - .../hdds/scm/container/ContainerReplicaCount.java | 10 + .../hdds/scm/container/ReplicationManager.java | 62 +- .../scm/container/ReplicationManagerReport.java | 192 +++++ ...etGreedy.java => AbstractFindTargetGreedy.java} | 87 +- .../scm/container/balancer/ContainerBalancer.java | 90 +- .../balancer/ContainerBalancerConfiguration.java | 218 +++-- .../FindTargetGreedyByNetworkTopology.java | 79 ++ .../balancer/FindTargetGreedyByUsageInfo.java | 52 ++ .../replication/ReplicationManagerMetrics.java | 42 + .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 35 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +- .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 24 + ...inerLocationProtocolServerSideTranslatorPB.java | 45 +- .../hdds/scm/server/SCMClientProtocolServer.java | 45 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 3 +- .../hdds/scm/server/StorageContainerManager.java | 9 +- .../server/StorageContainerManagerHttpServer.java | 4 +- .../hdds/scm/container/TestReplicationManager.java | 161 +++- .../container/TestReplicationManagerReport.java | 113 +++ .../container/balancer/TestContainerBalancer.java | 119 +-- .../container/balancer/TestFindTargetStrategy.java | 196 +++++ .../replication/TestReplicationManagerMetrics.java | 97 +++ .../states/TestContainerReplicaCount.java | 10 + .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 10 +- .../server/TestSCMUpdateServiceGrpcServer.java | 8 +- .../TestSCMHAUnfinalizedStateValidationAction.java | 1 + .../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 11 + .../hdds/scm/cli/ContainerBalancerCommands.java | 19 +- .../scm/cli/ContainerBalancerStartSubcommand.java | 50 +- .../hdds/scm/cli/ContainerOperationClient.java | 6 +- .../datanode/TestContainerBalancerSubCommand.java | 8 +- hadoop-ozone/client/pom.xml | 14 + .../apache/hadoop/ozone/client/OzoneBucket.java | 11 + .../checksum/AbstractBlockChecksumComputer.java} | 36 +- .../client/checksum/BaseFileChecksumHelper.java | 200 +++++ .../checksum/ReplicatedBlockChecksumComputer.java | 72 ++ .../checksum/ReplicatedFileChecksumHelper.java | 187 +++++ .../ozone/client/checksum/package-info.java} | 32 +- .../hadoop/ozone/client/io/KeyInputStream.java | 8 +- .../ozone/client/protocol/ClientProtocol.java | 10 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 33 +- .../TestReplicatedBlockChecksumComputer.java | 68 ++ .../checksum/TestReplicatedFileChecksumHelper.java | 323 ++++++++ .../ozone/client/checksum/package-info.java} | 32 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 94 ++- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 7 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 3 +- .../hadoop/ozone/om/helpers/OMNodeDetails.java | 25 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 36 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 10 +- .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 17 +- .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 4 + .../hadoop/ozone/om/protocol/OMAdminProtocol.java | 6 + .../hadoop/ozone/om/protocol/OMConfiguration.java | 30 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 12 + .../protocolPB/OMAdminProtocolClientSideImpl.java | 112 ++- ...OzoneManagerProtocolClientSideTranslatorPB.java | 22 + hadoop-ozone/dev-support/checks/acceptance.sh | 5 +- hadoop-ozone/dev-support/checks/coverage.sh | 1 - hadoop-ozone/dev-support/checks/kubernetes.sh | 4 +- hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/assemblies/ozone-src.xml | 1 + .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../dist/src/main/compose/ozone-ha/docker-config | 4 + .../dist/src/main/compose/ozone/docker-config | 1 + .../src/main/compose/ozonesecure-ha/docker-config | 2 +- .../compose/ozonesecure-ha/keytabs/HTTP.keytab | Bin 144 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/dn.keytab | Bin 278 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/om.keytab | Bin 278 -> 0 bytes .../compose/ozonesecure-ha/keytabs/recon.keytab | Bin 296 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/s3g.keytab | Bin 434 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/scm.keytab | Bin 586 -> 0 bytes .../compose/ozonesecure-ha/keytabs/testuser.keytab | Bin 152 -> 0 bytes .../ozonesecure-ha/keytabs/testuser2.keytab | Bin 154 -> 0 bytes .../src/main/compose/ozonesecure-mr/docker-config | 3 +- .../compose/ozonesecure-mr/keytabs/HTTP.keytab | Bin 144 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/dn.keytab | Bin 278 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/om.keytab | Bin 278 -> 0 bytes .../compose/ozonesecure-mr/keytabs/recon.keytab | Bin 296 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/s3g.keytab | Bin 434 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/scm.keytab | Bin 586 -> 0 bytes .../compose/ozonesecure-mr/keytabs/testuser.keytab | Bin 152 -> 0 bytes .../ozonesecure-mr/keytabs/testuser2.keytab | Bin 154 -> 0 bytes .../src/main/compose/ozonesecure/docker-config | 4 +- .../main/compose/ozonesecure/keytabs/HTTP.keytab | Bin 144 -> 0 bytes .../src/main/compose/ozonesecure/keytabs/dn.keytab | Bin 278 -> 0 bytes .../src/main/compose/ozonesecure/keytabs/om.keytab | Bin 278 -> 0 bytes .../main/compose/ozonesecure/keytabs/recon.keytab | Bin 296 -> 0 bytes .../main/compose/ozonesecure/keytabs/s3g.keytab | Bin 434 -> 0 bytes .../main/compose/ozonesecure/keytabs/scm.keytab | Bin 586 -> 0 bytes .../compose/ozonesecure/keytabs/testuser.keytab | Bin 152 -> 0 bytes .../compose/ozonesecure/keytabs/testuser2.keytab | Bin 154 -> 0 bytes .../main/compose/upgrade/compose/ha/docker-config | 4 +- .../compose/upgrade/compose/non-ha/docker-config | 2 +- hadoop-ozone/dist/src/main/docker/Dockerfile | 1 - .../dist/src/main/dockerlibexec/entrypoint.sh | 2 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 5 +- .../dist/src/main/smoketest/s3/awss3.robot | 8 + .../dist/src/main/smoketest/s3/objectcopy.robot | 20 +- .../dist/src/main/smoketest/s3/objectdelete.robot | 32 +- .../src/main/smoketest/s3/objectmultidelete.robot | 24 +- .../dist/src/main/smoketest/s3/objectputget.robot | 42 +- hadoop-ozone/dist/src/shell/ozone/ozone | 17 - .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 39 +- hadoop-ozone/insight/pom.xml | 1 - hadoop-ozone/integration-test/pom.xml | 10 - .../hadoop/fs/ozone/TestOzoneFileSystem.java | 2 + .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +- .../contract/ITestOzoneContractDistCpWithFSO.java | 65 ++ .../hadoop/fs/ozone/contract/OzoneContract.java | 18 +- .../metrics/TestSCMContainerManagerMetrics.java | 2 + .../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 + .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 + .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 4 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 16 - .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 368 ++++++--- .../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 116 --- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 23 + .../ozone/TestContainerBalancerOperations.java | 19 +- .../hadoop/ozone/TestMiniOzoneOMHACluster.java | 6 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 + .../hadoop/ozone/TestSecureOzoneCluster.java | 15 +- .../hadoop/ozone/TestStorageContainerManager.java | 3 + .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 7 +- .../client/rpc/TestContainerStateMachine.java | 5 +- .../rpc/TestContainerStateMachineFailures.java | 85 ++ .../rpc/TestContainerStateMachineFlushDelay.java | 4 +- .../client/rpc/TestOzoneAtRestEncryption.java | 27 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 22 + .../TestOzoneRpcClientWithKeyLatestVersion.java | 209 +++++ .../commandhandler/TestBlockDeletion.java | 1 + .../transport/server/ratis/TestCSMMetrics.java | 7 + .../ozoneimpl/TestSecureOzoneContainer.java | 2 + .../server/TestSecureContainerServer.java | 3 + .../ozone/dn/ratis/TestDnRatisLogParser.java | 2 + .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 1 + ...otstrap.java => TestAddRemoveOzoneManager.java} | 107 ++- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 361 ++++---- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 6 +- .../hadoop/ozone/om/TestOMUpgradeFinalization.java | 6 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 8 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +- .../ozone/om/TestOzoneManagerHAWithData.java | 2 +- .../ozone/om/TestOzoneManagerHAWithFailover.java | 2 +- .../om/multitenant/TestMultiTenantVolume.java | 28 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 8 +- .../hadoop/ozone/recon/TestReconScmHASnapshot.java | 70 ++ .../hadoop/ozone/recon/TestReconScmSnapshot.java | 133 +++ .../ozone/recon/TestReconWithOzoneManagerHA.java | 12 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 18 +- .../scm/node/TestDecommissionAndMaintenance.java | 6 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 4 +- .../src/main/proto/OMAdminProtocol.proto | 29 +- .../src/main/proto/OmClientProtocol.proto | 4 +- .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 2 +- .../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 36 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 111 --- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 912 --------------------- .../org/apache/hadoop/ozone/om/OzoneManager.java | 140 +++- .../hadoop/ozone/om/OzoneManagerHttpServer.java | 4 +- .../apache/hadoop/ozone/om/fs/OzoneManagerFS.java | 6 - .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 3 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 74 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 9 +- .../om/request/bucket/OMBucketSetOwnerRequest.java | 199 +++++ .../om/request/file/OMDirectoryCreateRequest.java | 2 +- .../file/OMDirectoryCreateRequestWithFSO.java | 2 +- .../ozone/om/request/file/OMFileCreateRequest.java | 6 +- .../request/file/OMFileCreateRequestWithFSO.java | 5 +- .../ozone/om/request/file/OMFileRequest.java | 25 + .../om/request/key/OMAllocateBlockRequest.java | 4 +- .../request/key/OMAllocateBlockRequestWithFSO.java | 4 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 2 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 2 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 6 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 32 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 35 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 3 +- .../om/request/key/OMPathsPurgeRequestWithFSO.java | 2 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 2 +- .../request/key/acl/OMKeyAddAclRequestWithFSO.java | 3 +- .../key/acl/OMKeyRemoveAclRequestWithFSO.java | 3 +- .../request/key/acl/OMKeySetAclRequestWithFSO.java | 3 +- .../S3InitiateMultipartUploadRequest.java | 4 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 19 +- .../multipart/S3MultipartUploadAbortRequest.java | 4 +- .../S3MultipartUploadAbortRequestWithFSO.java | 4 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../S3MultipartUploadCommitPartRequestWithFSO.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 12 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 20 +- .../OMBucketSetOwnerResponse.java} | 71 +- .../response/file/OMDirectoryCreateResponse.java | 10 +- .../file/OMDirectoryCreateResponseWithFSO.java | 15 +- .../om/response/file/OMFileCreateResponse.java | 6 +- .../response/file/OMFileCreateResponseWithFSO.java | 5 +- .../response/key/AbstractOMKeyDeleteResponse.java | 5 +- .../om/response/key/OMAllocateBlockResponse.java | 13 +- .../key/OMAllocateBlockResponseWithFSO.java | 14 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 6 +- .../response/key/OMKeyCommitResponseWithFSO.java | 5 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 6 +- .../response/key/OMKeyCreateResponseWithFSO.java | 6 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 6 +- .../response/key/OMKeyDeleteResponseWithFSO.java | 5 +- .../ozone/om/response/key/OMKeyPurgeResponse.java | 3 +- .../ozone/om/response/key/OMKeyRenameResponse.java | 5 +- .../response/key/OMKeyRenameResponseWithFSO.java | 5 +- .../om/response/key/OMKeysDeleteResponse.java | 6 +- .../om/response/key/OMOpenKeysDeleteRequest.java | 2 +- .../om/response/key/OMOpenKeysDeleteResponse.java | 6 +- .../response/key/OMPathsPurgeResponseWithFSO.java | 5 +- .../ozone/om/response/key/OmKeyResponse.java | 25 - .../om/response/key/acl/OMKeyAclResponse.java | 17 +- .../response/key/acl/OMKeyAclResponseWithFSO.java | 16 +- .../S3InitiateMultipartUploadResponse.java | 14 +- .../S3InitiateMultipartUploadResponseWithFSO.java | 10 +- .../multipart/S3MultipartUploadAbortResponse.java | 18 +- .../S3MultipartUploadAbortResponseWithFSO.java | 13 +- .../S3MultipartUploadCommitPartResponse.java | 14 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 9 +- .../S3MultipartUploadCompleteResponse.java | 40 +- .../S3MultipartUploadCompleteResponseWithFSO.java | 11 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 7 + .../protocolPB/OMAdminProtocolServerSideImpl.java | 50 ++ .../protocolPB/OzoneManagerRequestHandler.java | 1 + .../hadoop/ozone/security/AWSV4AuthValidator.java | 10 - .../org/apache/hadoop/ozone/om/OmTestManagers.java | 122 +++ .../hadoop/ozone/om/TestKeyDeletingService.java | 99 ++- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 56 +- .../apache/hadoop/ozone/om/TestTrashService.java | 34 +- .../volume/TestOMVolumeSetOwnerRequest.java | 7 +- .../volume/TestOMVolumeSetQuotaRequest.java | 7 +- .../ozone/om/response/TestCleanupTableInfo.java | 7 +- .../file/TestOMDirectoryCreateResponse.java | 2 +- .../file/TestOMDirectoryCreateResponseWithFSO.java | 4 +- .../response/key/TestOMAllocateBlockResponse.java | 2 +- .../key/TestOMAllocateBlockResponseWithFSO.java | 2 +- .../s3/multipart/TestS3MultipartResponse.java | 13 +- .../TestS3MultipartUploadAbortResponse.java | 20 +- .../TestS3MultipartUploadAbortResponseWithFSO.java | 22 +- .../security/acl/TestOzoneNativeAuthorizer.java | 77 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 44 +- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 41 +- .../apache/hadoop/ozone/recon/ReconConstants.java | 2 + .../hadoop/ozone/recon/ReconServerConfigKeys.java | 20 +- .../ozone/recon/fsck/ContainerHealthTask.java | 2 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 11 + .../scm/ReconStorageContainerManagerFacade.java | 129 ++- .../recon/spi/StorageContainerServiceProvider.java | 12 + .../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +- .../impl/StorageContainerServiceProviderImpl.java | 139 +++- .../TestStorageContainerServiceProviderImpl.java | 12 + .../hadoop/ozone/s3/OzoneClientProducer.java | 10 +- .../hadoop/ozone/s3/OzoneServiceProvider.java | 94 --- .../hadoop/ozone/s3/commontypes/CommonPrefix.java | 2 + .../hadoop/ozone/s3/commontypes/KeyMetadata.java | 1 + ...CommonPrefix.java => ObjectKeyNameAdapter.java} | 37 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 19 +- .../s3/signature/AuthorizationV4QueryParser.java | 5 +- .../ozone/s3/signature/StringToSignProducer.java | 5 +- .../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 81 -- .../org/apache/hadoop/ozone/s3/util/S3Utils.java} | 36 +- .../hadoop/ozone/s3/TestOzoneClientProducer.java | 33 + .../s3/commontypes/TestObjectKeyNameAdapter.java | 43 + .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 15 +- .../hadoop/ozone/s3/util/TestOzoneS3Util.java | 136 --- .../tools/dev-support/findbugsExcludeFile.xml | 4 - hadoop-ozone/tools/pom.xml | 15 - .../ozone/admin/om/DecommissionOMSubcommand.java | 212 +++++ .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 7 +- .../hadoop/ozone/genesis/BenchMarkCRCBatch.java | 141 ---- .../ozone/genesis/BenchMarkCRCStreaming.java | 173 ---- .../ozone/genesis/BenchMarkContainerStateMap.java | 199 ----- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 339 -------- .../ozone/genesis/BenchMarkOzoneManager.java | 193 ----- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 126 --- .../ozone/genesis/BenchmarkBlockDataToString.java | 166 ---- .../ozone/genesis/BenchmarkChunkManager.java | 180 ---- .../org/apache/hadoop/ozone/genesis/Genesis.java | 108 --- .../ozone/genesis/GenesisMemoryProfiler.java | 61 -- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 162 ---- .../apache/hadoop/ozone/genesis/package-info.java | 25 - .../hadoop/ozone/shell/bucket/BucketCommands.java | 3 +- .../ozone/shell/bucket/UpdateBucketHandler.java | 62 ++ pom.xml | 20 +- 345 files changed, 6658 insertions(+), 5538 deletions(-) diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java index cd7a58a,0000000..ee2d9ab mode 100644,000000..100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/multitenant/TestMultiTenantVolume.java @@@ -1,158 -1,0 +1,144 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.om.multitenant; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.client.HddsClientUtils; +import org.apache.hadoop.ozone.MiniOzoneCluster; - import org.apache.hadoop.ozone.MiniOzoneClusterProvider; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.rpc.RpcClient; +import org.apache.hadoop.ozone.om.OMMultiTenantManagerImpl; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.protocol.S3Auth; - import org.junit.After; ++import org.junit.AfterClass; +import org.junit.Assert; - import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; - import org.junit.jupiter.api.AfterAll; + +import java.util.UUID; + +/** + * Tests that S3 requests for a tenant are directed to that tenant's volume, + * and that users not belonging to a tenant are directed to the default S3 + * volume. + */ +public class TestMultiTenantVolume { - private static MiniOzoneClusterProvider clusterProvider; - private MiniOzoneCluster cluster; ++ private static MiniOzoneCluster cluster; + private static String s3VolumeName; + + @BeforeClass - public static void initClusterProvider() { ++ public static void initClusterProvider() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + conf.setBoolean( + OMMultiTenantManagerImpl.OZONE_OM_TENANT_DEV_SKIP_RANGER, true); + MiniOzoneCluster.Builder builder = MiniOzoneCluster.newBuilder(conf) + .withoutDatanodes(); - clusterProvider = new MiniOzoneClusterProvider(conf, builder, 2); ++ cluster = builder.build(); + s3VolumeName = HddsClientUtils.getDefaultS3VolumeName(conf); + } + - @Before - public void setup() throws Exception { - cluster = clusterProvider.provide(); - } - - @After - public void teardown() throws Exception { - clusterProvider.destroy(cluster); - } - - @AfterAll - public static void shutdownClusterProvider() throws Exception { - clusterProvider.shutdown(); ++ @AfterClass ++ public static void shutdownClusterProvider() { ++ cluster.shutdown(); + } + + @Test + public void testDefaultS3Volume() throws Exception { + final String bucketName = "bucket"; + + // Default client not belonging to a tenant should end up in the S3 volume. + ObjectStore store = cluster.getClient().getObjectStore(); + Assert.assertEquals(s3VolumeName, store.getS3Volume().getName()); + + // Create bucket. + store.createS3Bucket(bucketName); + Assert.assertEquals(s3VolumeName, + store.getS3Bucket(bucketName).getVolumeName()); + + // Delete bucket. + store.deleteS3Bucket(bucketName); + assertS3BucketNotFound(store, bucketName); + } + + @Test + public void testS3TenantVolume() throws Exception { + final String tenant = "tenant"; + final String principal = "username"; + final String bucketName = "bucket"; + final String accessID = UUID.randomUUID().toString(); + + ObjectStore store = getStoreForAccessID(accessID); + store.createTenant(tenant); + store.tenantAssignUserAccessId(principal, tenant, accessID); + + // S3 volume pointed to by the store should be for the tenant. + Assert.assertEquals(tenant, store.getS3Volume().getName()); + + // Create bucket in the tenant volume. + store.createS3Bucket(bucketName); + OzoneBucket bucket = store.getS3Bucket(bucketName); + Assert.assertEquals(tenant, bucket.getVolumeName()); + + // A different user should not see bucket, since they will be directed to + // the s3 volume. + ObjectStore store2 = getStoreForAccessID(UUID.randomUUID().toString()); + assertS3BucketNotFound(store2, bucketName); + + // Delete bucket. + store.deleteS3Bucket(bucketName); + assertS3BucketNotFound(store, bucketName); + } + + /** + * Checks that the bucket is not found using + * {@link ObjectStore#getS3Bucket} and the designated S3 volume pointed to + * by the ObjectStore. + */ + private void assertS3BucketNotFound(ObjectStore store, String bucketName) + throws Exception { + try { + store.getS3Bucket(bucketName); + } catch(OMException ex) { + if (ex.getResult() != OMException.ResultCodes.BUCKET_NOT_FOUND) { + throw ex; + } + } + + try { + OzoneVolume volume = store.getS3Volume(); + volume.getBucket(bucketName); + } catch(OMException ex) { + if (ex.getResult() != OMException.ResultCodes.BUCKET_NOT_FOUND) { + throw ex; + } + } + } + + private ObjectStore getStoreForAccessID(String accessID) throws Exception { + // Cluster provider will modify our provided configuration. We must use + // this version to build the client. + OzoneConfiguration conf = cluster.getOzoneManager().getConfiguration(); + // Manually construct an object store instead of using the cluster + // provided one so we can specify the access ID. + RpcClient client = new RpcClient(conf, null); + client.setTheadLocalS3Auth(new S3Auth("unused1", "unused2", accessID)); + return new ObjectStore(conf, client); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
