This is an automated email from the ASF dual-hosted git repository. umamahesh pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit e9b621521095160e5422ff2c760244005a1957fd Merge: df96b86c8d 55d5ecbda2 Author: Uma Maheswara Rao G <[email protected]> AuthorDate: Mon Apr 11 10:52:51 2022 -0700 Merge master to EC Branch HDDS-3816-ec .../java/org/apache/hadoop/hdds/HddsUtils.java | 1 + .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../protocol/StorageContainerLocationProtocol.java | 10 +- .../ContainerCommandResponseBuilders.java | 13 +- .../org/apache/hadoop/ozone/lock/ActiveLock.java | 35 +- .../org/apache/hadoop/ozone/lock/LockManager.java | 49 +- .../ozone/container/ContainerTestHelper.java | 28 + .../common/statemachine/DatanodeStateMachine.java | 31 ++ .../common/statemachine/StateContext.java | 13 + .../commandhandler/CommandDispatcher.java | 16 + .../commandhandler/CommandHandler.java | 10 + .../commandhandler/DeleteBlocksCommandHandler.java | 11 + .../DeleteContainerCommandHandler.java | 5 + .../ReplicateContainerCommandHandler.java | 5 + .../states/endpoint/HeartbeatEndpointTask.java | 21 + .../ozone/container/keyvalue/KeyValueHandler.java | 40 +- .../container/keyvalue/impl/BlockManagerImpl.java | 10 +- .../replication/ReplicationSupervisor.java | 2 +- .../container/common/impl/TestHddsDispatcher.java | 20 + .../common/statemachine/TestStateContext.java | 36 ++ .../states/endpoint/TestHeartbeatEndpointTask.java | 26 +- .../container/keyvalue/TestKeyValueHandler.java | 7 +- hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 + hadoop-hdds/docs/content/interface/S3.md | 7 - hadoop-hdds/docs/content/interface/S3.zh.md | 7 - .../docs/content/start/StartFromDockerHub.md | 6 - .../docs/content/start/StartFromDockerHub.zh.md | 5 - ...inerLocationProtocolClientSideTranslatorPB.java | 15 +- .../hadoop/hdds/server/http/HttpServer2.java | 1 + .../src/main/proto/ScmAdminProtocol.proto | 1 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 6 + .../scm/container/balancer/ContainerBalancer.java | 397 ++++++++----- .../balancer/ContainerBalancerMetrics.java | 50 +- .../IllegalContainerBalancerStateException.java | 46 ++ ...lidContainerBalancerConfigurationException.java | 47 ++ ...inerLocationProtocolServerSideTranslatorPB.java | 17 +- .../hdds/scm/server/SCMClientProtocolServer.java | 40 +- .../hdds/scm/server/StorageContainerManager.java | 2 +- .../container/balancer/TestContainerBalancer.java | 155 ++++-- .../scm/cli/ContainerBalancerStartSubcommand.java | 14 +- .../hdds/scm/cli/ContainerOperationClient.java | 8 +- .../datanode/TestContainerBalancerSubCommand.java | 22 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 1 + .../ozone/client/protocol/ClientProtocol.java | 1 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 1 + .../apache/hadoop/ozone/om/lock/LockUsageInfo.java | 63 +++ .../apache/hadoop/ozone/om/lock/OMLockMetrics.java | 207 +++++++ .../hadoop/ozone/om/lock/OzoneManagerLock.java | 273 ++++++++- ...OzoneManagerProtocolClientSideTranslatorPB.java | 1 + .../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 227 ++++++++ .../dist/src/main/compose/xcompat/docker-config | 3 +- .../fs/ozone/TestOzoneFileSystemMissingParent.java | 3 + .../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 245 ++++++++ .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 6 +- .../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 48 ++ .../rooted/ITestRootedOzoneContractRootDir.java | 1 - .../TestContainerStateManagerIntegration.java | 89 +-- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 4 + .../container/ozoneimpl/TestOzoneContainer.java | 2 - .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 81 +++ .../om/ratis/TestOzoneManagerRatisRequest.java | 35 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 56 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 11 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 144 +++-- .../apache/hadoop/ozone/om/OzoneManagerUtils.java | 72 ++- .../om/ratis/utils/OzoneManagerRatisUtils.java | 77 ++- .../BucketLayoutAwareOMKeyRequestFactory.java | 314 +++++++++++ .../ozone/om/request/OMClientRequestUtils.java | 50 ++ .../ozone/om/request/OMKeyRequestFactory.java | 139 ----- .../om/request/bucket/OMBucketCreateRequest.java | 6 +- .../om/request/file/OMDirectoryCreateRequest.java | 19 - .../ozone/om/request/file/OMFileCreateRequest.java | 17 - .../ozone/om/request/file/OMFileRequest.java | 56 +- .../om/request/key/OMAllocateBlockRequest.java | 14 - .../ozone/om/request/key/OMKeyCommitRequest.java | 15 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 18 - .../ozone/om/request/key/OMKeyDeleteRequest.java | 15 - .../om/request/key/OMKeyDeleteRequestWithFSO.java | 4 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 19 - .../hadoop/ozone/om/request/key/OMKeyRequest.java | 7 + .../ozone/om/request/key/OMKeysDeleteRequest.java | 137 +++-- .../ozone/om/request/key/OMKeysRenameRequest.java | 5 +- .../om/request/key/OMOpenKeysDeleteRequest.java | 6 +- .../om/request/key/OmKeysDeleteRequestWithFSO.java | 141 +++++ .../S3InitiateMultipartUploadRequest.java | 15 - .../multipart/S3MultipartUploadAbortRequest.java | 15 - .../S3MultipartUploadCommitPartRequest.java | 15 - .../S3MultipartUploadCompleteRequest.java | 27 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 3 +- .../hadoop/ozone/om/response/OMClientResponse.java | 16 - .../om/response/key/OMKeysDeleteResponse.java | 8 + .../response/key/OMKeysDeleteResponseWithFSO.java | 94 ++++ .../org/apache/hadoop/ozone/om/OmTestManagers.java | 12 +- .../ozone/om/ScmBlockLocationTestingClient.java | 12 + .../hadoop/ozone/om/TestKeyDeletingService.java | 135 +++-- .../request/TestBucketLayoutAwareOMKeyFactory.java | 166 ++++++ .../request/file/TestOMDirectoryCreateRequest.java | 37 +- .../file/TestOMDirectoryCreateRequestWithFSO.java | 16 +- .../om/request/file/TestOMFileCreateRequest.java | 6 +- .../file/TestOMFileCreateRequestWithFSO.java | 2 +- .../om/request/key/TestOMKeyCreateRequest.java | 7 +- .../om/request/key/TestOMKeyRenameRequest.java | 14 +- .../om/request/key/TestOMKeysDeleteRequest.java | 35 +- .../key/TestOMKeysDeleteRequestWithFSO.java | 109 ++++ .../om/request/key/TestOMKeysRenameRequest.java | 4 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 4 +- .../TestS3InitiateMultipartUploadRequest.java | 2 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 2 +- .../TestS3MultipartUploadAbortRequest.java | 4 +- .../TestS3MultipartUploadAbortRequestWithFSO.java | 5 + .../TestS3MultipartUploadCommitPartRequest.java | 6 +- .../TestS3MultipartUploadCompleteRequest.java | 10 +- ...estS3MultipartUploadCompleteRequestWithFSO.java | 7 +- .../upgrade/TestOMCancelPrepareRequest.java | 2 +- .../ozone/om/response/TestCleanupTableInfo.java | 6 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 133 ++++- hadoop-ozone/ozonefs-shaded/pom.xml | 6 + hadoop-ozone/pom.xml | 2 +- .../hadoop/ozone/recon/tasks/OMDBUpdateEvent.java | 2 +- .../ozone/recon/tasks/OMDBUpdatesHandler.java | 37 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 252 ++++++--- hadoop-ozone/s3gateway/pom.xml | 4 + .../hadoop/ozone/s3/S3GatewayHttpServer.java | 19 + .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 10 - .../s3gateway/src/main/resources/browser.html | 617 --------------------- .../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 2 +- .../hadoop/ozone/s3/endpoint/TestBucketList.java | 36 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 4 +- pom.xml | 7 +- 129 files changed, 4035 insertions(+), 1765 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java index f014251109,e2a5c8c750..dd5690359e --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/client/ScmClient.java @@@ -19,7 -19,7 +19,8 @@@ package org.apache.hadoop.hdds.scm.clie import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.annotation.InterfaceStability; +import org.apache.hadoop.hdds.client.ReplicationConfig; + import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto; import org.apache.hadoop.hdds.scm.DatanodeAdminError; import org.apache.hadoop.hdds.scm.container.ContainerReplicaInfo; import org.apache.hadoop.hdds.scm.container.ReplicationManagerReport; diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java index 1df588e5f5,1b07c5d3d6..619f8353fb --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/protocol/StorageContainerLocationProtocol.java @@@ -18,8 -18,8 +18,9 @@@ package org.apache.hadoop.hdds.scm.protocol; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto; import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type; import org.apache.hadoop.hdds.scm.DatanodeAdminError; import org.apache.hadoop.hdds.scm.ScmConfig; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index d7edcea17c,223986b5ec..a7fdef0bcd --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@@ -29,11 -28,8 +28,10 @@@ import java.util.Map import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneAcl; - import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 0bf7f3ade2,39f090bcdd..824a78d13a --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@@ -26,8 -25,6 +25,7 @@@ import java.util.Map import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.client.ReplicationConfig; - import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java index 52dff6294a,26a68b89f4..8a90246d15 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRequest.java @@@ -51,8 -49,9 +51,9 @@@ import org.apache.hadoop.ozone.om.helpe import org.apache.hadoop.ozone.om.helpers.OmPrefixInfo; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; -import org.apache.hadoop.ozone.om.helpers.BucketLayout; +import org.apache.hadoop.ozone.om.helpers.QuotaUtil; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; + import org.apache.hadoop.ozone.om.request.OMClientRequestUtils; import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.protocolPB.OMPBHelper; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index 2c5e25df36,c54faf6815..97002898e6 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@@ -23,9 -23,7 +23,8 @@@ import com.google.common.base.Precondit import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneConfigUtil; import org.apache.hadoop.ozone.om.OzoneManager; - import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java index 6823a92e28,cb5bb230c9..f0af5e7e55 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequest.java @@@ -485,10 -424,9 +485,9 @@@ public class TestOMKeyCreateRequest ext // Add volume and bucket entries to DB. addVolumeAndBucketToDB(volumeName, bucketName, - omMetadataManager); - + omMetadataManager, getBucketLayout()); - keyName = "dir1/dir2/dir3/file1"; + String keyName = "dir1/dir2/dir3/file1"; createAndCheck(keyName); // Key with leading '/'. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
