This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 723cdaaaca36620d72a930265056dd14a534f6c8 Merge: 974136d 61a237a Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Sat May 22 13:01:25 2021 +0530 Merge remote-tracking branch 'origin/master' into HDDS-2939 .github/workflows/post-commit.yml | 33 +- .gitignore | 3 + CONTRIBUTING.md | 10 +- NOTICE.txt | 2 +- README.md | 2 +- .../hadoop/hdds/scm/XceiverClientFactory.java | 3 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 8 +- hadoop-hdds/common/pom.xml | 5 + .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 119 ++++--- .../hdds/ratis/ServerNotLeaderException.java | 113 ++++++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 +- .../hadoop/hdds/scm/exceptions/SCMException.java | 3 +- .../hadoop/hdds/scm/ha/NonRetriableException.java | 30 ++ .../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 4 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 102 +++++- .../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 5 +- .../protocol/StorageContainerLocationProtocol.java | 18 + .../security/token/ContainerTokenIdentifier.java | 103 ++++++ .../security/token/OzoneBlockTokenIdentifier.java | 146 +++----- .../security/token/ShortLivedTokenIdentifier.java | 112 ++++++ .../hadoop/hdds/security/x509/SecurityConfig.java | 16 + .../hadoop/hdds/security/x509/crl}/CRLCodec.java | 2 +- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 1 - .../hdds/security/x509/crl}/CRLInfoCodec.java | 3 +- .../hdds/security/x509/crl/package-info.java | 24 ++ .../org/apache/hadoop/hdds/utils/db/Codec.java | 0 .../apache/hadoop/hdds/utils/db/package-info.java | 22 ++ .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 12 + .../common/src/main/resources/ozone-default.xml | 172 ++++++--- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 22 +- .../hdds/ratis/TestServerNotLeaderException.java | 82 +++++ .../ozone/container/ContainerTestHelper.java | 45 +-- .../hdds/datanode/metadata/CRLDBDefinition.java | 99 ++++++ .../hdds/datanode/metadata/DatanodeCRLStore.java | 84 +++++ .../datanode/metadata/DatanodeCRLStoreImpl.java | 127 +++++++ .../hdds/datanode/metadata/package-info.java | 22 ++ .../apache/hadoop/ozone/HddsDatanodeService.java | 19 +- .../container/common/impl/HddsDispatcher.java | 25 +- .../common/statemachine/DatanodeConfiguration.java | 30 ++ .../common/statemachine/DatanodeStateMachine.java | 2 +- .../common/states/datanode/InitDatanodeState.java | 4 +- .../states/endpoint/HeartbeatEndpointTask.java | 2 +- .../states/endpoint/RegisterEndpointTask.java | 2 +- .../container/common/utils/HddsVolumeUtil.java | 11 + .../ozone/container/common/volume/HddsVolume.java | 13 + .../container/common/volume/MutableVolumeSet.java | 28 +- .../container/keyvalue/KeyValueContainer.java | 6 + .../container/keyvalue/KeyValueContainerData.java | 2 + .../ozone/container/keyvalue/KeyValueHandler.java | 3 +- .../container/keyvalue/helpers/BlockUtils.java | 4 +- .../container/keyvalue/helpers/ChunkUtils.java | 42 ++- .../keyvalue/impl/ChunkManagerDispatcher.java | 10 +- .../keyvalue/impl/ChunkManagerFactory.java | 5 +- .../keyvalue/impl/FilePerBlockStrategy.java | 32 +- .../keyvalue/impl/FilePerChunkStrategy.java | 15 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 21 +- .../metadata/TestDatanodeCRLStoreImpl.java | 127 +++++++ .../hdds/datanode/metadata/package-info.java} | 31 +- .../hadoop/ozone/TestHddsDatanodeService.java | 6 +- .../container/common/TestBlockDeletingService.java | 47 ++- .../TestSchemaOneBackwardsCompatibility.java | 18 + .../impl/TestContainerDeletionChoosingPolicy.java | 31 +- .../common/impl/TestContainerPersistence.java | 3 +- .../statemachine/TestDatanodeConfiguration.java | 15 + .../container/keyvalue/ChunkLayoutTestInfo.java | 4 +- .../container/keyvalue/helpers/TestChunkUtils.java | 14 +- .../keyvalue/impl/TestFilePerChunkStrategy.java | 3 +- hadoop-hdds/docs/content/interface/ReconApi.md | 390 +++++++++------------ hadoop-hdds/docs/content/interface/ReconApi.zh.md | 356 ++++++++----------- hadoop-hdds/framework/pom.xml | 6 + .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 11 + .../SCMSecurityProtocolClientSideTranslatorPB.java | 14 + .../hadoop/hdds/scm/protocolPB/OzonePBHelper.java | 82 +++++ ...lockLocationProtocolClientSideTranslatorPB.java | 6 - ...inerLocationProtocolClientSideTranslatorPB.java | 64 +++- .../SCMBlockLocationFailoverProxyProvider.java | 150 +++++--- .../hadoop/hdds/scm/proxy/SCMClientConfig.java | 27 +- .../SCMContainerLocationFailoverProxyProvider.java | 185 +++++----- .../SCMSecurityProtocolFailoverProxyProvider.java | 97 +++-- .../hadoop/hdds}/security/OzoneSecretKey.java | 4 +- .../hadoop/hdds}/security/OzoneSecretManager.java | 2 +- .../hdds}/security/OzoneSecurityException.java | 2 +- .../apache/hadoop/hdds/security/package-info.java | 31 +- .../hdds/security/token/BlockTokenVerifier.java | 149 +++----- .../security/token/CompositeTokenVerifier.java | 42 ++- .../token/ContainerTokenSecretManager.java | 48 +++ .../security/token/ContainerTokenVerifier.java | 51 +++ .../hdds/security/token/NoopTokenVerifier.java | 38 +- .../token/ShortLivedTokenSecretManager.java | 107 ++++++ .../security/token/ShortLivedTokenVerifier.java | 134 +++++++ .../hadoop/hdds/security/token/TokenVerifier.java | 63 +++- .../certificate/authority/CertificateServer.java | 4 +- .../certificate/authority/DefaultCAServer.java | 3 +- .../certificate/authority/DefaultCRLApprover.java | 2 +- .../x509/certificate/client/CertificateClient.java | 15 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 2 + .../hadoop/hdds/utils/DBCheckpointServlet.java | 30 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 162 +++++---- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 45 +-- .../security/token/TestBlockTokenVerifier.java | 75 ++++ .../security/token/TestContainerTokenVerifier.java | 75 ++++ .../token/TestOzoneBlockTokenIdentifier.java | 2 +- .../hdds/security/token/TokenVerifierTests.java | 242 +++++++++++++ .../certificate/authority/TestDefaultCAServer.java | 6 +- .../x509/certificate/utils/TestCRLCodec.java | 1 + .../src/main/proto/ScmAdminProtocol.proto | 31 ++ .../interface-client/src/main/proto/hdds.proto | 16 + .../src/main/proto/ScmServerProtocol.proto | 1 + .../src/main/proto/ScmServerSecurityProtocol.proto | 28 ++ .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 16 +- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 51 ++- .../scm/container/ContainerStateManagerImpl.java | 11 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 30 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 22 ++ .../hadoop/hdds/scm/ha/MockSCMHAManager.java | 16 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 18 + .../apache/hadoop/hdds/scm/ha/SCMHAManager.java | 29 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 192 +++++----- .../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 7 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 53 ++- .../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 5 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 19 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 94 +++-- .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 1 + .../hdds/scm/node/NodeDecommissionManager.java | 31 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 33 +- ...lockLocationProtocolServerSideTranslatorPB.java | 19 +- ...inerLocationProtocolServerSideTranslatorPB.java | 58 ++- .../hdds/scm/server/SCMClientProtocolServer.java | 92 +++-- .../hdds/scm/server/SCMDBCheckpointServlet.java | 5 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 33 +- .../hdds/scm/server/StorageContainerManager.java | 153 ++++++-- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 31 +- .../hadoop/hdds/scm/TestHddsServerUtils.java | 21 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 21 +- .../algorithms/TestContainerPlacementFactory.java | 51 ++- .../TestSCMContainerPlacementCapacity.java | 37 +- .../TestSCMContainerPlacementRackAware.java | 105 ++++-- .../TestSCMContainerPlacementRandom.java | 32 +- .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 76 ++++ .../hdds/scm/node/TestNodeDecommissionManager.java | 33 ++ .../scm/pipeline/TestRatisPipelineProvider.java | 4 +- .../hdds/scm/cli/ContainerOperationClient.java | 30 +- .../apache/hadoop/ozone/client/ObjectStore.java | 4 + .../apache/hadoop/ozone/client/OzoneBucket.java | 15 +- .../hadoop/ozone/client/OzoneKeyLocation.java | 15 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 57 ++- .../client/io/BlockOutputStreamEntryPool.java | 2 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 5 +- .../ozone/client/protocol/ClientProtocol.java | 7 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 50 ++- .../hadoop/ozone/client/MockBlockAllocator.java} | 36 +- .../hadoop/ozone/client/MockDatanodeStorage.java | 73 ++++ .../hadoop/ozone/client/MockOmTransport.java | 276 +++++++++++++++ .../ozone/client/MockXceiverClientFactory.java | 72 ++++ .../hadoop/ozone/client/MockXceiverClientSpi.java | 180 ++++++++++ .../ozone/client/SinglePipelineBlockAllocator.java | 90 +++++ .../hadoop/ozone/client/TestHddsClientUtils.java | 150 ++++---- .../hadoop/ozone/client/TestOzoneClient.java | 174 +++++++++ .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../org/apache/hadoop/ozone/audit/OMAction.java | 3 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 39 +-- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 20 ++ .../ozone/om/protocol/OzoneManagerProtocol.java | 8 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 11 + .../ozone/security/acl/IAccessAuthorizer.java | 1 - hadoop-ozone/dev-support/checks/_lib.sh | 28 ++ hadoop-ozone/dev-support/checks/build.sh | 2 +- hadoop-ozone/dev-support/checks/dependency.sh | 50 +++ .../dev-support/checks/{build.sh => docs.sh} | 27 +- hadoop-ozone/dev-support/checks/junit.sh | 2 +- .../src/main/compose/compatibility/docker-config | 1 + .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../src/main/compose/ozone-ha/docker-compose.yaml | 10 +- .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../src/main/compose/ozone-om-ha/docker-config | 1 + .../src/main/compose/ozone-topology/docker-config | 1 + .../dist/src/main/compose/ozone/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/test.sh | 1 + .../src/main/compose/ozoneblockade/docker-config | 1 + .../src/main/compose/ozones3-haproxy/docker-config | 1 + .../src/main/compose/ozonescripts/docker-config | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 16 +- .../src/main/compose/ozonesecure-ha/docker-config | 4 +- .../dist/src/main/compose/ozonesecure-ha/test.sh | 5 + .../src/main/compose/ozonesecure-mr/docker-config | 2 + .../src/main/compose/ozonesecure/docker-config | 2 + .../dist/src/main/compose/restart/docker-config | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 4 +- .../dist/src/main/compose/upgrade/docker-config | 1 + .../dist/src/main/compose/xcompat/docker-config | 1 + hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 235 +++++++++++++ .../test.sh => license/update-jar-report.sh} | 28 +- .../src/main/smoketest/admincli/container.robot | 14 +- .../src/main/smoketest/admincli/datanode.robot | 6 +- .../src/main/smoketest/admincli/pipeline.robot | 7 +- .../smoketest/admincli/replicationmanager.robot | 7 +- .../src/main/smoketest/admincli/safemode.robot | 6 +- .../dist/src/main/smoketest/admincli/scmha.robot | 2 +- .../dist/src/main/smoketest/cli/envvars.robot | 15 + .../dist/src/main/smoketest/commonlib.robot | 11 + .../dist/src/main/smoketest/freon/remove.robot | 43 +++ hadoop-ozone/dist/src/shell/ozone/ozone | 7 +- .../dist/src/shell/ozone/ozone-functions.sh | 3 +- .../network-tests/src/test/compose/docker-config | 1 + .../hdds/scm/TestSCMDbCheckpointServlet.java | 9 +- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 12 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 7 +- .../org/apache/hadoop/ozone/RatisTestHelper.java | 33 ++ .../hadoop/ozone/TestOzoneConfigurationFields.java | 5 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 50 ++- .../hadoop/ozone/TestStorageContainerManager.java | 32 +- .../ozone/client/rpc/Test2WayCommitInRatis.java | 3 +- .../TestContainerStateMachineFailureOnRead.java | 6 +- .../client/rpc/TestDeleteWithSlowFollower.java | 10 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 24 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 12 - .../ozone/client/rpc/TestWatchForCommit.java | 3 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 11 +- .../ozoneimpl/TestSecureOzoneContainer.java | 151 ++++---- .../server/TestSecureContainerServer.java | 183 +++++----- .../apache/hadoop/ozone/dn/DatanodeTestUtils.java | 243 +++++++++++++ .../TestDatanodeHddsVolumeFailureDetection.java | 289 +++++++++++++++ .../ozone/freon/TestFreonWithDatanodeRestart.java | 126 ------- .../ozone/freon/TestHadoopDirTreeGenerator.java | 2 +- .../ozone/freon/TestHadoopNestedDirGenerator.java | 5 +- .../ozone/freon/TestOzoneClientKeyGenerator.java | 94 ----- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 19 + .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 274 +++++++++------ .../ozone/om/TestOzoneManagerRestInterface.java | 3 +- .../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +- .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 147 ++++++++ .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 73 ++-- .../src/main/proto/OmClientProtocol.proto | 12 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 8 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 27 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 117 +++++-- .../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 3 + .../om/request/bucket/OMBucketDeleteRequest.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 9 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 2 +- .../om/request/s3/security/S3GetSecretRequest.java | 11 +- .../request/s3/security/S3RevokeSecretRequest.java | 155 ++++++++ .../s3/security/S3RevokeSecretResponse.java | 57 +++ ...OzoneManagerProtocolServerSideTranslatorPB.java | 27 +- .../security/OzoneBlockTokenSecretManager.java | 86 +---- .../OzoneDelegationTokenSecretManager.java | 1 + .../security/TestOzoneBlockTokenSecretManager.java | 285 ++++++++------- hadoop-ozone/ozonefs-hadoop2/pom.xml | 10 - hadoop-ozone/ozonefs-hadoop3/pom.xml | 10 - hadoop-ozone/ozonefs/pom.xml | 10 - .../ozone/recon/scm/ReconContainerManager.java | 107 ++++-- .../recon/scm/ReconContainerReportHandler.java | 24 +- .../ReconIncrementalContainerReportHandler.java | 2 +- .../recon/spi/StorageContainerServiceProvider.java | 8 + .../impl/StorageContainerServiceProviderImpl.java | 6 + .../ozone/recon/api/TestContainerEndpoint.java | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 8 + .../scm/AbstractReconContainerManagerTest.java | 30 ++ .../ozone/recon/scm/TestReconContainerManager.java | 50 ++- ...TestReconIncrementalContainerReportHandler.java | 3 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 17 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 283 +++++++++++++-- .../s3/endpoint/ListMultipartUploadsResult.java | 53 +-- .../endpoint/PutBucketAclRequestUnmarshaller.java | 85 +++++ .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 342 ++++++++++++++++++ .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 235 +++++++++++++ .../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 72 ++++ .../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 + .../hadoop/ozone/client/OzoneBucketStub.java | 23 ++ .../hadoop/ozone/client/OzoneVolumeStub.java | 24 ++ .../hadoop/ozone/s3/TestOzoneClientProducer.java | 4 +- .../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 266 ++++++++++++++ .../{TestBucketGet.java => TestBucketList.java} | 76 ++-- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 6 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 56 ++- .../src/test/resources/groupAccessControlList.xml | 57 +++ .../src/test/resources/userAccessControlList.xml | 42 +++ .../StorageContainerManagersCommandHandler.java | 2 +- .../org/apache/hadoop/ozone/debug/OzoneDebug.java | 10 + .../ContainerCommands.java} | 130 +++---- .../ozone/debug/container/ExportSubcommand.java | 75 ++++ .../ozone/debug/container/InfoSubcommand.java | 53 +++ .../container/ListSubcommand.java} | 53 ++- .../hadoop/ozone/debug/container/package-info.java | 22 ++ .../hadoop/ozone/freon/BaseFreonGenerator.java | 7 + .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 2 +- .../java/org/apache/hadoop/ozone/freon/Freon.java | 2 + .../hadoop/ozone/freon/OmBucketGenerator.java | 2 +- ...OmBucketGenerator.java => OmBucketRemover.java} | 36 +- ...etGenerator.java => OzoneClientKeyRemover.java} | 60 ++-- .../hadoop/ozone/freon/RandomKeyGenerator.java | 137 ++++++++ .../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 + .../containergenerator/GeneratorDatanode.java | 2 +- .../GenerateOzoneRequiredConfigurations.java | 44 ++- .../ozone/genesis/BenchmarkChunkManager.java | 4 +- .../hadoop/ozone/shell/s3/GetS3SecretHandler.java | 11 +- ...cretHandler.java => RevokeS3SecretHandler.java} | 57 +-- .../org/apache/hadoop/ozone/shell/s3/S3Shell.java | 3 +- .../TestGenerateOzoneRequiredConfigurations.java | 52 ++- pom.xml | 44 +-- 305 files changed, 10352 insertions(+), 3222 deletions(-) diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 5fedd83,577146d..f524cfb --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@@ -289,7 -289,7 +289,8 @@@ public final class OmUtils case PurgeKeys: case RecoverTrash: case DeleteOpenKeys: + case RevokeS3Secret: + case PurgePaths: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); diff --cc hadoop-ozone/dist/src/main/compose/ozone/test.sh index 2b55188,b8d86ac..7695acc --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@@ -48,29 -54,8 +48,30 @@@ execute_robot_test scm om-rati execute_robot_test scm freon execute_robot_test scm cli + execute_robot_test scm admincli + +execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:bucket -N ozonefs-simple-ofs-bucket ozonefs/ozonefs.robot +execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:link -N ozonefs-simple-o3fs-link ozonefs/ozonefs.robot + +# running FS tests with different config requires restart of the cluster +export OZONE_KEEP_RESULTS=true +stop_docker_env + +## Restarting the cluster with prefix-layout enabled (FSO) +export OZONE_OM_METADATA_LAYOUT=PREFIX +export OZONE_OM_ENABLE_FILESYSTEM_PATHS=true +start_docker_env + +execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-prefix-ofs-link ozonefs/ozonefs.robot +execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-prefix-o3fs-bucket ozonefs/ozonefs.robot + +execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-prefix-layout-objectputget s3/objectputget.robot +execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-prefix-layout-objectdelete s3/objectdelete.robot +execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-prefix-layout-objectcopy s3/objectcopy.robot +execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-prefix-layout-objectmultidelete s3/objectmultidelete.robot +execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-prefix-layout-MultipartUpload s3/MultipartUpload.robot + stop_docker_env generate_report diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 3bacb0e,a10bb67..295d2ad --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -94,7 -94,7 +94,9 @@@ enum Type ListTrash = 91; RecoverTrash = 92; - PurgePaths = 93; + RevokeS3Secret = 93; ++ ++ PurgePaths = 94; } message OMRequest { @@@ -168,7 -168,7 +170,9 @@@ optional ListTrashRequest listTrashRequest = 91; optional RecoverTrashRequest RecoverTrashRequest = 92; - optional PurgePathsRequest purgePathsRequest = 93; + optional RevokeS3SecretRequest RevokeS3SecretRequest = 93; ++ ++ optional PurgePathsRequest purgePathsRequest = 94; } message OMResponse { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index ea83ded,982d229..1abd2d3 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -217,11 -218,9 +220,12 @@@ import static org.apache.hadoop.ozone.o import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HANDLER_COUNT_KEY; + import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_AUTH_TYPE; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_DEFAULT; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METADATA_LAYOUT_PREFIX; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_METRICS_SAVE_INTERVAL_DEFAULT; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_USER_MAX_VOLUME; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index e4a08a0,29a0328..e95cd33 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@@ -60,14 -49,11 +60,15 @@@ import org.apache.hadoop.ozone.om.reque import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixRemoveAclRequest; import org.apache.hadoop.ozone.om.request.key.acl.prefix.OMPrefixSetAclRequest; import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequest; +import org.apache.hadoop.ozone.om.request.s3.multipart.S3InitiateMultipartUploadRequestWithFSO; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequest; +import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadAbortRequestWithFSO; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequest; +import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCommitPartRequestWithFSO; import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCompleteRequest; +import org.apache.hadoop.ozone.om.request.s3.multipart.S3MultipartUploadCompleteRequestWithFSO; import org.apache.hadoop.ozone.om.request.s3.security.S3GetSecretRequest; + import org.apache.hadoop.ozone.om.request.s3.security.S3RevokeSecretRequest; import org.apache.hadoop.ozone.om.request.security.OMCancelDelegationTokenRequest; import org.apache.hadoop.ozone.om.request.security.OMGetDelegationTokenRequest; import org.apache.hadoop.ozone.om.request.security.OMRenewDelegationTokenRequest; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
