This is an automated email from the ASF dual-hosted git repository. umamahesh pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 2e60a4f8cb0e8594b1e4b46ef99f8c68cfd85fe2 Merge: 4d660c4 11605e1 Author: Uma Maheswara Rao G <[email protected]> AuthorDate: Mon Jan 31 13:08:55 2022 -0800 Merge master to EC Branch .gitignore | 1 + CONTRIBUTING.md | 3 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 2 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 4 +- hadoop-hdds/common/pom.xml | 15 + .../apache/hadoop/hdds/DFSConfigKeysLegacy.java | 4 + .../hadoop/hdds/conf/OzoneConfiguration.java | 10 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 24 +- .../hdds/scm/container/ContainerReplicaInfo.java | 129 +++ .../scm/container/ReplicationManagerReport.java | 283 +++++++ .../protocol/StorageContainerLocationProtocol.java | 24 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 +- .../org/apache/hadoop/ozone/audit/SCMAction.java | 3 +- .../common/src/main/resources/ozone-default.xml | 109 ++- .../hadoop/hdds/conf/TestOzoneConfiguration.java | 28 + .../scm/container/TestContainerReplicaInfo.java | 59 ++ .../container/TestReplicationManagerReport.java | 162 ++++ .../hadoop/hdds/scm/container/package-info.java | 5 +- .../hadoop/hdds/tracing/TestStringCodec.java | 18 +- ...TestUpgradeUtils.java => UpgradeTestUtils.java} | 4 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 6 +- .../common/helpers/DatanodeVersionFile.java | 8 +- .../ozone/container/common/impl/ContainerData.java | 19 +- .../container/common/impl/ContainerDataYaml.java | 11 +- ...OutVersion.java => ContainerLayoutVersion.java} | 34 +- .../ozone/container/common/interfaces/Handler.java | 2 +- .../common/statemachine/DatanodeConfiguration.java | 32 - .../common/statemachine/DatanodeStateMachine.java | 7 +- .../CloseContainerCommandHandler.java | 17 +- .../common/transport/server/XceiverServerGrpc.java | 2 +- .../transport/server/ratis/XceiverServerRatis.java | 2 +- .../container/keyvalue/KeyValueContainerCheck.java | 10 +- .../container/keyvalue/KeyValueContainerData.java | 8 +- .../ozone/container/keyvalue/KeyValueHandler.java | 6 +- .../keyvalue/impl/ChunkManagerDispatcher.java | 18 +- .../keyvalue/impl/FilePerBlockStrategy.java | 4 +- .../keyvalue/impl/FilePerChunkStrategy.java | 4 +- .../ozoneimpl/ContainerScrubberConfiguration.java | 65 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 11 + .../container/replication/ContainerDownloader.java | 3 +- .../replication/DownloadAndImportReplicator.java | 19 +- .../replication/GrpcReplicationClient.java | 15 +- .../container/replication/ReplicationServer.java | 56 +- .../replication/ReplicationSupervisor.java | 8 + .../replication/SimpleContainerDownloader.java | 39 +- .../ozone/container/common/ContainerTestUtils.java | 4 +- .../container/common/TestBlockDeletingService.java | 13 +- ...ersion.java => TestContainerLayoutVersion.java} | 12 +- .../common/TestKeyValueContainerData.java | 10 +- .../common/impl/TestContainerDataYaml.java | 16 +- .../impl/TestContainerDeletionChoosingPolicy.java | 8 +- .../common/impl/TestContainerPersistence.java | 8 +- .../container/common/impl/TestContainerSet.java | 8 +- .../container/common/impl/TestHddsDispatcher.java | 8 +- .../statemachine/TestDatanodeConfiguration.java | 11 - .../TestCloseContainerCommandHandler.java | 14 +- .../common/volume/TestStorageVolumeChecker.java | 8 +- ...tTestInfo.java => ContainerLayoutTestInfo.java} | 24 +- .../keyvalue/TestKeyValueBlockIterator.java | 10 +- .../container/keyvalue/TestKeyValueContainer.java | 16 +- .../keyvalue/TestKeyValueContainerCheck.java | 9 +- .../TestKeyValueContainerMarkUnhealthy.java | 8 +- .../container/keyvalue/TestKeyValueHandler.java | 8 +- .../container/keyvalue/TestTarContainerPacker.java | 8 +- .../keyvalue/impl/AbstractTestChunkManager.java | 8 +- .../keyvalue/impl/TestBlockManagerImpl.java | 10 +- .../keyvalue/impl/TestChunkManagerDummyImpl.java | 6 +- .../keyvalue/impl/TestFilePerBlockStrategy.java | 6 +- .../keyvalue/impl/TestFilePerChunkStrategy.java | 12 +- .../container/ozoneimpl/TestContainerReader.java | 8 +- .../TestContainerScrubberConfiguration.java | 97 +++ .../container/ozoneimpl/TestOzoneContainer.java | 10 +- .../replication/TestReplicationConfig.java | 75 ++ .../replication/TestReplicationSupervisor.java | 14 +- .../replication/TestSimpleContainerDownloader.java | 13 +- .../upgrade/TestDataNodeStartupSlvLessThanMlv.java | 4 +- hadoop-hdds/docs/config.yaml | 5 + hadoop-hdds/docs/content/concept/Containers.md | 2 +- hadoop-hdds/docs/content/concept/Datanodes.md | 4 +- hadoop-hdds/docs/content/concept/OzoneManager.md | 6 +- hadoop-hdds/docs/content/concept/Recon.md | 6 +- hadoop-hdds/docs/content/feature/OM-HA.md | 5 +- hadoop-hdds/docs/content/feature/Observability.md | 2 +- .../{Observability.md => Observability.zh.md} | 63 +- hadoop-hdds/docs/content/feature/PrefixFSO.md | 8 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +- hadoop-hdds/docs/content/security/SecureOzone.md | 10 +- .../docs/content/security/SecureOzone.zh.md | 34 +- .../docs/content/security/SecuringDatanodes.md | 2 +- .../docs/content/security/SecuringDatanodes.zh.md | 2 +- hadoop-hdds/docs/content/start/OnPrem.md | 1 + hadoop-hdds/docs/content/start/OnPrem.zh.md | 2 + hadoop-hdds/docs/content/tools/TestTools.md | 129 +-- hadoop-hdds/docs/content/tools/TestTools.zh.md | 129 +-- hadoop-hdds/docs/content/tools/_index.md | 1 - hadoop-hdds/docs/content/tools/_index.zh.md | 1 - .../themes/ozonedoc/layouts/shortcodes/image.html | 19 + .../java/org/apache/hadoop/hdds/ExitManager.java | 4 + ...inerLocationProtocolClientSideTranslatorPB.java | 86 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +- .../org/apache/hadoop/hdds/utils/db/DBStore.java | 10 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 11 + .../certificate/authority/TestDefaultCAServer.java | 6 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 24 + .../src/main/proto/ScmAdminProtocol.proto | 40 +- .../interface-client/src/main/proto/hdds.proto | 25 + hadoop-hdds/server-scm/pom.xml | 10 - .../hdds/scm/container/ContainerReplicaCount.java | 10 + .../hdds/scm/container/ReplicationManager.java | 67 +- ...etGreedy.java => AbstractFindTargetGreedy.java} | 87 +- .../scm/container/balancer/ContainerBalancer.java | 90 +- .../balancer/ContainerBalancerConfiguration.java | 218 +++-- .../FindTargetGreedyByNetworkTopology.java | 79 ++ .../balancer/FindTargetGreedyByUsageInfo.java | 52 ++ .../replication/ReplicationManagerMetrics.java | 42 + .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 35 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 3 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +- .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 24 + ...inerLocationProtocolServerSideTranslatorPB.java | 78 +- .../hdds/scm/server/SCMClientProtocolServer.java | 77 +- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 3 +- .../hdds/scm/server/StorageContainerManager.java | 9 +- .../server/StorageContainerManagerHttpServer.java | 4 +- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 692 +++++++++++++++- .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 699 ---------------- .../hadoop/hdds/scm/block/TestBlockManager.java | 18 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 4 +- .../command/TestCommandStatusReportHandler.java | 4 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 6 +- .../container/TestCloseContainerEventHandler.java | 4 +- .../scm/container/TestContainerReportHandler.java | 4 +- .../TestIncrementalContainerReportHandler.java | 4 +- .../hdds/scm/container/TestReplicationManager.java | 165 +++- .../scm/container/TestUnknownContainerReport.java | 2 +- .../container/balancer/TestContainerBalancer.java | 119 +-- .../container/balancer/TestFindTargetStrategy.java | 196 +++++ .../algorithms/TestContainerPlacementFactory.java | 12 +- .../TestSCMContainerPlacementCapacity.java | 12 +- .../TestSCMContainerPlacementRackAware.java | 22 +- .../TestSCMContainerPlacementRackScatter.java | 22 +- .../TestSCMContainerPlacementRandom.java | 16 +- .../replication/TestReplicationManagerMetrics.java | 97 +++ .../states/TestContainerReplicaCount.java | 10 + .../hdds/scm/crl/TestCRLStatusReportHandler.java | 4 +- .../hdds/scm/node/TestContainerPlacement.java | 6 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 43 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 4 +- .../hdds/scm/node/TestNodeReportHandler.java | 10 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 78 +- .../hdds/scm/node/TestSCMNodeStorageStatMap.java | 15 +- .../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 16 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 11 +- .../scm/server/TestSCMBlockProtocolServer.java | 4 +- .../server/TestSCMUpdateServiceGrpcServer.java | 8 +- .../TestSCMHAUnfinalizedStateValidationAction.java | 5 +- .../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 4 +- .../ozone/container/common/TestEndPoint.java | 23 +- .../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 11 +- hadoop-hdds/tools/pom.xml | 9 + .../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 11 + .../hdds/scm/cli/ContainerBalancerCommands.java | 19 +- .../scm/cli/ContainerBalancerStartSubcommand.java | 50 +- .../hdds/scm/cli/ContainerOperationClient.java | 34 +- .../hdds/scm/cli/container/ContainerCommands.java | 3 +- .../hdds/scm/cli/container/InfoSubcommand.java | 61 +- .../hdds/scm/cli/container/ReportSubcommand.java | 116 +++ .../hdds/scm/cli/container/TestInfoSubCommand.java | 249 ++++++ .../scm/cli/container/TestReportSubCommand.java | 159 ++++ .../datanode/TestContainerBalancerSubCommand.java | 8 +- hadoop-ozone/client/pom.xml | 14 + .../apache/hadoop/ozone/client/OzoneBucket.java | 11 + .../checksum/AbstractBlockChecksumComputer.java} | 36 +- .../client/checksum/BaseFileChecksumHelper.java | 200 +++++ .../checksum/ReplicatedBlockChecksumComputer.java | 72 ++ .../checksum/ReplicatedFileChecksumHelper.java | 187 +++++ .../ozone/client/checksum}/package-info.java | 8 +- .../hadoop/ozone/client/io/KeyInputStream.java | 8 +- .../ozone/client/protocol/ClientProtocol.java | 27 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 96 ++- .../hadoop/ozone/client/MockOmTransport.java | 4 + .../TestReplicatedBlockChecksumComputer.java | 68 ++ .../checksum/TestReplicatedFileChecksumHelper.java | 323 ++++++++ .../ozone/client/checksum}/package-info.java | 8 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 94 ++- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 13 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 3 +- .../hadoop/ozone/om/helpers/OMNodeDetails.java | 25 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 49 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 10 +- .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 17 +- .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 4 + .../hadoop/ozone/om/protocol/OMAdminProtocol.java | 6 + .../hadoop/ozone/om/protocol/OMConfiguration.java | 30 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 12 + .../protocolPB/OMAdminProtocolClientSideImpl.java | 112 ++- ...OzoneManagerProtocolClientSideTranslatorPB.java | 22 + hadoop-ozone/dev-support/checks/acceptance.sh | 5 +- hadoop-ozone/dev-support/checks/coverage.sh | 1 - hadoop-ozone/dev-support/checks/kubernetes.sh | 4 +- hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/assemblies/ozone-src.xml | 1 + .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../dist/src/main/compose/ozone-ha/docker-config | 4 + .../dist/src/main/compose/ozone/docker-config | 1 + .../src/main/compose/ozonesecure-ha/docker-config | 2 +- .../compose/ozonesecure-ha/keytabs/HTTP.keytab | Bin 144 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/dn.keytab | Bin 278 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/om.keytab | Bin 278 -> 0 bytes .../compose/ozonesecure-ha/keytabs/recon.keytab | Bin 296 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/s3g.keytab | Bin 434 -> 0 bytes .../main/compose/ozonesecure-ha/keytabs/scm.keytab | Bin 586 -> 0 bytes .../compose/ozonesecure-ha/keytabs/testuser.keytab | Bin 152 -> 0 bytes .../ozonesecure-ha/keytabs/testuser2.keytab | Bin 154 -> 0 bytes .../src/main/compose/ozonesecure-mr/docker-config | 3 +- .../compose/ozonesecure-mr/keytabs/HTTP.keytab | Bin 144 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/dn.keytab | Bin 278 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/om.keytab | Bin 278 -> 0 bytes .../compose/ozonesecure-mr/keytabs/recon.keytab | Bin 296 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/s3g.keytab | Bin 434 -> 0 bytes .../main/compose/ozonesecure-mr/keytabs/scm.keytab | Bin 586 -> 0 bytes .../compose/ozonesecure-mr/keytabs/testuser.keytab | Bin 152 -> 0 bytes .../ozonesecure-mr/keytabs/testuser2.keytab | Bin 154 -> 0 bytes .../src/main/compose/ozonesecure/docker-config | 4 +- .../main/compose/ozonesecure/keytabs/HTTP.keytab | Bin 144 -> 0 bytes .../src/main/compose/ozonesecure/keytabs/dn.keytab | Bin 278 -> 0 bytes .../src/main/compose/ozonesecure/keytabs/om.keytab | Bin 278 -> 0 bytes .../main/compose/ozonesecure/keytabs/recon.keytab | Bin 296 -> 0 bytes .../main/compose/ozonesecure/keytabs/s3g.keytab | Bin 434 -> 0 bytes .../main/compose/ozonesecure/keytabs/scm.keytab | Bin 586 -> 0 bytes .../compose/ozonesecure/keytabs/testuser.keytab | Bin 152 -> 0 bytes .../compose/ozonesecure/keytabs/testuser2.keytab | Bin 154 -> 0 bytes .../main/compose/upgrade/compose/ha/docker-config | 4 +- .../compose/upgrade/compose/non-ha/docker-config | 2 +- hadoop-ozone/dist/src/main/docker/Dockerfile | 1 - .../dist/src/main/dockerlibexec/entrypoint.sh | 2 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 5 +- .../dist/src/main/smoketest/s3/awss3.robot | 8 + .../dist/src/main/smoketest/s3/objectcopy.robot | 20 +- .../dist/src/main/smoketest/s3/objectdelete.robot | 32 +- .../src/main/smoketest/s3/objectmultidelete.robot | 24 +- .../dist/src/main/smoketest/s3/objectputget.robot | 42 +- hadoop-ozone/dist/src/shell/ozone/ozone | 17 - .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 39 +- hadoop-ozone/insight/pom.xml | 1 - hadoop-ozone/integration-test/pom.xml | 10 - .../hadoop/fs/ozone/TestOzoneFileSystem.java | 2 + .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +- .../contract/ITestOzoneContractDistCpWithFSO.java | 65 ++ .../hadoop/fs/ozone/contract/OzoneContract.java | 20 +- .../metrics/TestSCMContainerManagerMetrics.java | 2 + .../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 + .../hdds/scm/pipeline/TestPipelineClose.java | 4 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 + .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 4 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 22 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 374 ++++++--- .../hadoop/ozone/MiniOzoneOMHAClusterImpl.java | 116 --- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 23 + .../ozone/TestContainerBalancerOperations.java | 19 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 2 +- .../hadoop/ozone/TestMiniOzoneOMHACluster.java | 6 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 + .../hadoop/ozone/TestSecureOzoneCluster.java | 37 +- .../hadoop/ozone/TestStandardOutputUtil.java | 84 ++ .../hadoop/ozone/TestStorageContainerManager.java | 11 +- .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 7 +- .../client/rpc/TestContainerStateMachine.java | 5 +- .../rpc/TestContainerStateMachineFailures.java | 218 ++++- .../rpc/TestContainerStateMachineFlushDelay.java | 4 +- .../client/rpc/TestDeleteWithSlowFollower.java | 11 +- .../client/rpc/TestOzoneAtRestEncryption.java | 39 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 48 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 115 +-- .../TestOzoneRpcClientWithKeyLatestVersion.java | 209 +++++ .../hadoop/ozone/client/rpc/TestReadRetries.java | 18 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +- .../client/rpc/read/TestChunkInputStream.java | 4 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 15 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 4 +- .../apache/hadoop/ozone/container/TestHelper.java | 7 +- .../commandhandler/TestBlockDeletion.java | 1 + .../commandhandler/TestCloseContainerHandler.java | 4 +- .../commandhandler/TestDeleteContainerHandler.java | 3 +- .../transport/server/ratis/TestCSMMetrics.java | 7 + .../container/metrics/TestContainerMetrics.java | 3 +- .../ozoneimpl/TestSecureOzoneContainer.java | 2 + .../container/server/TestContainerServer.java | 3 +- .../server/TestSecureContainerServer.java | 6 +- .../ozone/dn/ratis/TestDnRatisLogParser.java | 2 + .../hadoop/ozone/dn/scrubber/TestDataScrubber.java | 7 +- .../TestDatanodeHddsVolumeFailureDetection.java | 7 +- .../freon/TestHadoopDirTreeGeneratorWithFSO.java | 6 +- .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 1 + ...otstrap.java => TestAddRemoveOzoneManager.java} | 107 ++- .../ozone/om/TestContainerReportWithKeys.java | 2 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 383 ++++----- .../org/apache/hadoop/ozone/om/TestKeyPurging.java | 2 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 6 +- .../hadoop/ozone/om/TestOMUpgradeFinalization.java | 6 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 4 +- .../ozone/om/TestOmStartupSlvLessThanMlv.java | 4 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 12 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 2 +- .../ozone/om/TestOzoneManagerHAWithData.java | 2 +- .../ozone/om/TestOzoneManagerHAWithFailover.java | 2 +- .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 2 +- .../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 5 +- .../om/ratis/TestOzoneManagerRatisRequest.java | 4 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 8 +- .../hadoop/ozone/recon/TestReconScmHASnapshot.java | 70 ++ .../hadoop/ozone/recon/TestReconScmSnapshot.java | 133 +++ .../ozone/recon/TestReconWithOzoneManager.java | 4 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 6 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 12 +- .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 2 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 18 +- .../ozone/scm/TestStorageContainerManagerHA.java | 4 +- .../scm/node/TestDecommissionAndMaintenance.java | 6 +- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 122 ++- .../hadoop/ozone/shell/TestOzoneShellHA.java | 4 +- .../src/main/proto/OMAdminProtocol.proto | 29 +- .../src/main/proto/OmClientProtocol.proto | 6 +- .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 2 +- .../hadoop/ozone/om/codec/TestOmKeyInfoCodec.java | 4 +- .../ozone/om/codec/TestRepeatedOmKeyInfoCodec.java | 40 +- .../ozone/om/helpers/TestInstanceHelper.java | 68 -- .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 39 +- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 57 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 111 --- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 912 --------------------- .../org/apache/hadoop/ozone/om/OzoneManager.java | 164 ++-- .../hadoop/ozone/om/OzoneManagerHttpServer.java | 4 +- .../apache/hadoop/ozone/om/OzoneManagerUtils.java | 55 ++ .../apache/hadoop/ozone/om/fs/OzoneManagerFS.java | 6 - .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 3 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 74 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 9 +- .../om/request/bucket/OMBucketCreateRequest.java | 1 - .../om/request/bucket/OMBucketSetOwnerRequest.java | 199 +++++ .../om/request/file/OMDirectoryCreateRequest.java | 2 +- .../file/OMDirectoryCreateRequestWithFSO.java | 2 +- .../ozone/om/request/file/OMFileCreateRequest.java | 6 +- .../request/file/OMFileCreateRequestWithFSO.java | 5 +- .../ozone/om/request/file/OMFileRequest.java | 25 + .../om/request/key/OMAllocateBlockRequest.java | 4 +- .../request/key/OMAllocateBlockRequestWithFSO.java | 4 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 2 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 2 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 6 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 32 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 3 +- .../om/request/key/OMKeyDeleteRequestWithFSO.java | 2 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 4 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 5 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 35 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 3 +- .../key/OMOpenKeysDeleteRequest.java | 6 +- .../om/request/key/OMPathsPurgeRequestWithFSO.java | 2 +- .../om/request/key/OMTrashRecoverRequest.java | 2 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 2 +- .../request/key/acl/OMKeyAddAclRequestWithFSO.java | 3 +- .../key/acl/OMKeyRemoveAclRequestWithFSO.java | 3 +- .../request/key/acl/OMKeySetAclRequestWithFSO.java | 3 +- .../S3InitiateMultipartUploadRequest.java | 4 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 19 +- .../multipart/S3MultipartUploadAbortRequest.java | 4 +- .../S3MultipartUploadAbortRequestWithFSO.java | 4 +- .../S3MultipartUploadCommitPartRequest.java | 2 +- .../S3MultipartUploadCommitPartRequestWithFSO.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 12 +- .../S3MultipartUploadCompleteRequestWithFSO.java | 20 +- .../OMBucketSetOwnerResponse.java} | 71 +- .../response/file/OMDirectoryCreateResponse.java | 10 +- .../file/OMDirectoryCreateResponseWithFSO.java | 15 +- .../om/response/file/OMFileCreateResponse.java | 6 +- .../response/file/OMFileCreateResponseWithFSO.java | 5 +- .../response/key/AbstractOMKeyDeleteResponse.java | 16 +- .../om/response/key/OMAllocateBlockResponse.java | 13 +- .../key/OMAllocateBlockResponseWithFSO.java | 14 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 11 +- .../response/key/OMKeyCommitResponseWithFSO.java | 5 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 11 +- .../response/key/OMKeyCreateResponseWithFSO.java | 6 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 8 +- .../response/key/OMKeyDeleteResponseWithFSO.java | 5 +- .../ozone/om/response/key/OMKeyPurgeResponse.java | 3 +- .../ozone/om/response/key/OMKeyRenameResponse.java | 18 +- .../response/key/OMKeyRenameResponseWithFSO.java | 9 +- .../om/response/key/OMKeysDeleteResponse.java | 6 +- .../om/response/key/OMOpenKeysDeleteResponse.java | 6 +- .../response/key/OMPathsPurgeResponseWithFSO.java | 13 +- .../om/response/key/OMTrashRecoverResponse.java | 8 +- .../ozone/om/response/key/OmKeyResponse.java | 35 +- .../om/response/key/acl/OMKeyAclResponse.java | 17 +- .../response/key/acl/OMKeyAclResponseWithFSO.java | 16 +- .../S3InitiateMultipartUploadResponse.java | 14 +- .../S3InitiateMultipartUploadResponseWithFSO.java | 10 +- .../multipart/S3MultipartUploadAbortResponse.java | 18 +- .../S3MultipartUploadAbortResponseWithFSO.java | 13 +- .../S3MultipartUploadCommitPartResponse.java | 14 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 9 +- .../S3MultipartUploadCompleteResponse.java | 40 +- .../S3MultipartUploadCompleteResponseWithFSO.java | 11 +- .../om/snapshot/OzoneManagerSnapshotProvider.java | 7 + .../protocolPB/OMAdminProtocolServerSideImpl.java | 50 ++ .../protocolPB/OzoneManagerRequestHandler.java | 1 + .../hadoop/ozone/security/AWSV4AuthValidator.java | 10 - .../org/apache/hadoop/ozone/om/OmTestManagers.java | 122 +++ .../hadoop/ozone/om/TestBucketManagerImpl.java | 6 +- .../hadoop/ozone/om/TestKeyDeletingService.java | 106 ++- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 70 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 36 +- .../apache/hadoop/ozone/om/TestTrashService.java | 40 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 8 +- ...OMRequestUtils.java => OMRequestTestUtils.java} | 30 +- .../request/TestOMClientRequestWithUserInfo.java | 2 +- .../ozone/om/request/bucket/TestBucketRequest.java | 1 - .../request/bucket/TestOMBucketCreateRequest.java | 8 +- .../bucket/TestOMBucketCreateRequestWithFSO.java | 7 +- .../request/bucket/TestOMBucketDeleteRequest.java | 6 +- .../bucket/TestOMBucketSetPropertyRequest.java | 8 +- .../bucket/acl/TestOMBucketAddAclRequest.java | 12 +- .../bucket/acl/TestOMBucketRemoveAclRequest.java | 14 +- .../bucket/acl/TestOMBucketSetAclRequest.java | 12 +- .../request/file/TestOMDirectoryCreateRequest.java | 22 +- .../file/TestOMDirectoryCreateRequestWithFSO.java | 49 +- .../om/request/file/TestOMFileCreateRequest.java | 30 +- .../file/TestOMFileCreateRequestWithFSO.java | 22 +- .../om/request/key/TestOMAllocateBlockRequest.java | 12 +- .../key/TestOMAllocateBlockRequestWithFSO.java | 10 +- .../ozone/om/request/key/TestOMKeyAclRequest.java | 10 +- .../om/request/key/TestOMKeyAclRequestWithFSO.java | 8 +- .../om/request/key/TestOMKeyCommitRequest.java | 12 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 8 +- .../om/request/key/TestOMKeyCreateRequest.java | 8 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 6 +- .../om/request/key/TestOMKeyDeleteRequest.java | 10 +- .../request/key/TestOMKeyDeleteRequestWithFSO.java | 10 +- .../key/TestOMKeyPurgeRequestAndResponse.java | 8 +- .../om/request/key/TestOMKeyRenameRequest.java | 15 +- .../om/request/key/TestOMKeysDeleteRequest.java | 6 +- .../om/request/key/TestOMKeysRenameRequest.java | 6 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 11 +- .../om/request/key/TestOMPrefixAclRequest.java | 6 +- .../TestS3InitiateMultipartUploadRequest.java | 6 +- ...estS3InitiateMultipartUploadRequestWithFSO.java | 4 +- .../s3/multipart/TestS3MultipartRequest.java | 12 +- .../TestS3MultipartUploadAbortRequest.java | 8 +- .../TestS3MultipartUploadAbortRequestWithFSO.java | 4 +- .../TestS3MultipartUploadCommitPartRequest.java | 12 +- ...tS3MultipartUploadCommitPartRequestWithFSO.java | 10 +- .../TestS3MultipartUploadCompleteRequest.java | 12 +- ...estS3MultipartUploadCompleteRequestWithFSO.java | 8 +- .../upgrade/TestOMCancelPrepareRequest.java | 2 +- .../request/volume/TestOMVolumeCreateRequest.java | 4 +- .../request/volume/TestOMVolumeDeleteRequest.java | 12 +- .../volume/TestOMVolumeSetOwnerRequest.java | 27 +- .../volume/TestOMVolumeSetQuotaRequest.java | 27 +- .../volume/acl/TestOMVolumeAddAclRequest.java | 12 +- .../volume/acl/TestOMVolumeRemoveAclRequest.java | 14 +- .../volume/acl/TestOMVolumeSetAclRequest.java | 12 +- .../ozone/om/response/TestCleanupTableInfo.java | 5 +- .../file/TestOMDirectoryCreateResponse.java | 6 +- .../file/TestOMDirectoryCreateResponseWithFSO.java | 8 +- .../file/TestOMFileCreateResponseWithFSO.java | 4 +- .../response/key/TestOMAllocateBlockResponse.java | 6 +- .../key/TestOMAllocateBlockResponseWithFSO.java | 6 +- .../om/response/key/TestOMKeyCommitResponse.java | 6 +- .../key/TestOMKeyCommitResponseWithFSO.java | 8 +- .../key/TestOMKeyCreateResponseWithFSO.java | 4 +- .../om/response/key/TestOMKeyDeleteResponse.java | 4 +- .../key/TestOMKeyDeleteResponseWithFSO.java | 12 +- .../om/response/key/TestOMKeyRenameResponse.java | 20 +- .../ozone/om/response/key/TestOMKeyResponse.java | 4 +- .../om/response/key/TestOMKeysDeleteResponse.java | 4 +- .../om/response/key/TestOMKeysRenameResponse.java | 8 +- .../response/key/TestOMOpenKeysDeleteResponse.java | 8 +- .../s3/multipart/TestS3MultipartResponse.java | 13 +- .../TestS3MultipartUploadAbortResponse.java | 20 +- .../TestS3MultipartUploadAbortResponseWithFSO.java | 22 +- ...S3MultipartUploadCommitPartResponseWithFSO.java | 10 +- ...stS3MultipartUploadCompleteResponseWithFSO.java | 18 +- .../security/acl/TestOzoneNativeAuthorizer.java | 83 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 50 +- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 47 +- .../apache/hadoop/ozone/recon/ReconConstants.java | 2 + .../hadoop/ozone/recon/ReconServerConfigKeys.java | 20 +- .../ozone/recon/fsck/ContainerHealthTask.java | 2 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 11 + .../scm/ReconStorageContainerManagerFacade.java | 129 ++- .../recon/spi/StorageContainerServiceProvider.java | 12 + .../spi/impl/OzoneManagerServiceProviderImpl.java | 8 +- .../impl/StorageContainerServiceProviderImpl.java | 139 +++- .../TestStorageContainerServiceProviderImpl.java | 12 + .../hadoop/ozone/s3/OzoneClientProducer.java | 9 +- .../hadoop/ozone/s3/OzoneServiceProvider.java | 94 --- .../hadoop/ozone/s3/commontypes/CommonPrefix.java | 2 + .../hadoop/ozone/s3/commontypes/KeyMetadata.java | 1 + ...CommonPrefix.java => ObjectKeyNameAdapter.java} | 37 +- .../ozone/s3/endpoint/CopyObjectResponse.java | 2 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 19 +- .../s3/signature/AuthorizationV4QueryParser.java | 5 +- .../ozone/s3/signature/StringToSignProducer.java | 5 +- .../apache/hadoop/ozone/s3/util/OzoneS3Util.java | 81 -- .../org/apache/hadoop/ozone/s3/util/S3Utils.java} | 36 +- .../hadoop/ozone/client/OzoneBucketStub.java | 2 +- .../hadoop/ozone/s3/TestOzoneClientProducer.java | 33 + .../s3/commontypes/TestObjectKeyNameAdapter.java | 43 + .../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 +- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 15 +- .../hadoop/ozone/s3/util/TestOzoneS3Util.java | 136 --- .../tools/dev-support/findbugsExcludeFile.xml | 4 - hadoop-ozone/tools/pom.xml | 15 - .../ozone/admin/nssummary/DiskUsageSubCommand.java | 2 +- .../admin/nssummary/FileSizeDistSubCommand.java | 2 +- .../ozone/admin/nssummary/NSSummaryAdmin.java | 36 +- .../ozone/admin/nssummary/NSSummaryCLIUtils.java | 8 +- .../admin/nssummary/QuotaUsageSubCommand.java | 2 +- .../ozone/admin/nssummary/SummarySubCommand.java | 2 +- .../ozone/admin/om/DecommissionOMSubcommand.java | 212 +++++ .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 7 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +- .../apache/hadoop/ozone/debug/PrefixParser.java | 12 +- .../apache/hadoop/ozone/debug/ReadReplicas.java | 247 ++++++ .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 8 +- .../hadoop/ozone/freon/RandomKeyGenerator.java | 54 +- .../containergenerator/GeneratorDatanode.java | 6 +- .../hadoop/ozone/genesis/BenchMarkCRCBatch.java | 141 ---- .../ozone/genesis/BenchMarkCRCStreaming.java | 173 ---- .../ozone/genesis/BenchMarkContainerStateMap.java | 199 ----- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 339 -------- .../ozone/genesis/BenchMarkOzoneManager.java | 193 ----- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 126 --- .../ozone/genesis/BenchmarkBlockDataToString.java | 166 ---- .../ozone/genesis/BenchmarkChunkManager.java | 180 ---- .../org/apache/hadoop/ozone/genesis/Genesis.java | 108 --- .../ozone/genesis/GenesisMemoryProfiler.java | 61 -- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 162 ---- .../apache/hadoop/ozone/genesis/package-info.java | 25 - .../hadoop/ozone/shell/bucket/BucketCommands.java | 3 +- .../ozone/shell/bucket/UpdateBucketHandler.java | 62 ++ pom.xml | 20 +- 546 files changed, 10427 insertions(+), 7442 deletions(-) diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java index e88df7c,2f459fb..8193747 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/scm/container/package-info.java @@@ -15,7 -15,7 +15,8 @@@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.hadoop.hdds.scm.container; ++ /** - Test cases for SCM Container classes. + Test cases for SCM container client classes. */ + package org.apache.hadoop.hdds.scm.container; diff --cc hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java index 67a21da,70efc40..c0843ab --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerDataYaml.java @@@ -27,9 -26,8 +27,9 @@@ import org.apache.hadoop.hdds.conf.Ozon import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; - import org.apache.hadoop.ozone.container.keyvalue.ChunkLayoutTestInfo; + import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.junit.Assert; import org.apache.hadoop.ozone.container.upgrade.VersionedDatanodeFeatures; import org.apache.ozone.test.GenericTestUtils; import org.junit.Test; diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java index 0d0b167,f440cf9..46a7b50 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyInputStream.java @@@ -364,9 -358,9 +364,9 @@@ public class KeyInputStream extends Ext } @Override - public void close() throws IOException { + public synchronized void close() throws IOException { closed = true; - for (BlockInputStream blockStream : blockStreams) { + for (ExtendedInputStream blockStream : blockStreams) { blockStream.close(); } } @@@ -405,8 -399,8 +405,8 @@@ } @Override - public void unbuffer() { + public synchronized void unbuffer() { - for (BlockInputStream is : blockStreams) { + for (ExtendedInputStream is : blockStreams) { is.unbuffer(); } } diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 5c20ef1,b9db113..429add3 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@@ -57,10 -58,10 +60,12 @@@ import org.apache.hadoop.hdds.scm.ScmCo import org.apache.hadoop.hdds.scm.XceiverClientFactory; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; + import org.apache.hadoop.hdds.scm.pipeline.Pipeline; + import org.apache.hadoop.hdds.scm.pipeline.PipelineID; import org.apache.hadoop.hdds.tracing.TracingUtil; import org.apache.hadoop.hdds.utils.IOUtils; +import org.apache.hadoop.io.ByteBufferPool; +import org.apache.hadoop.io.ElasticByteBufferPool; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConfigKeys; @@@ -287,11 -284,12 +293,15 @@@ public class RpcClient implements Clien } } }).build(); + this.byteBufferPool = new ElasticByteBufferPool(); + this.blockInputStreamFactory = BlockInputStreamFactoryImpl + .getInstance(byteBufferPool); } + public XceiverClientFactory getXceiverClientManager() { + return xceiverClientManager; + } + static boolean validateOmVersion(String expectedVersion, List<ServiceInfo> serviceInfoList) { if (expectedVersion == null || expectedVersion.isEmpty()) { diff --cc hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java index 9bc11e2,17a7f6b..c663970 --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java @@@ -83,6 -74,10 +83,10 @@@ public class MockOmTransport implement this.blockAllocator = allocator; } + public MockOmTransport() { - this.blockAllocator = new SinglePipelineBlockAllocator(); ++ this(new SinglePipelineBlockAllocator(new OzoneConfiguration())); + } + @Override public OMResponse submitRequest(OMRequest payload) throws IOException { switch (payload.getCmdType()) { diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java index 56e825f,1806a03..4932d5a --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java @@@ -52,7 -51,10 +52,11 @@@ public final class OmBucketArgs extend private long quotaInBytes; private long quotaInNamespace; + private DefaultReplicationConfig defaultReplicationConfig = null; + /** + * Bucket Owner Name. + */ + private String ownerName; /** * Private constructor, constructed via builder. @@@ -124,22 -129,14 +131,31 @@@ } /** + * Returns Bucket default replication config. + * @return + */ + public DefaultReplicationConfig getDefaultReplicationConfig() { + return defaultReplicationConfig; + } + + /** + * Sets the Bucket default replication config. + */ + private void setDefaultReplicationConfig( + DefaultReplicationConfig defaultReplicationConfig) { + this.defaultReplicationConfig = defaultReplicationConfig; + } + + /** + * Returns Bucket Owner Name. ++ * + * @return ownerName. + */ + public String getOwnerName() { + return ownerName; + } + + /** * Returns new builder class that builds a OmBucketArgs. * @return Builder */ @@@ -173,8 -173,7 +192,8 @@@ private Map<String, String> metadata; private long quotaInBytes; private long quotaInNamespace; + private DefaultReplicationConfig defaultReplicationConfig; - + private String ownerName; /** * Constructs a builder. */ @@@ -218,12 -217,11 +237,17 @@@ return this; } + public Builder setDefaultReplicationConfig( + DefaultReplicationConfig defaultRepConfig) { + this.defaultReplicationConfig = defaultRepConfig; + return this; + } + + public Builder setOwnerName(String owner) { + ownerName = owner; + return this; + } + /** * Constructs the OmBucketArgs. * @return instance of OmBucketArgs. @@@ -231,11 -229,8 +255,11 @@@ public OmBucketArgs build() { Preconditions.checkNotNull(volumeName); Preconditions.checkNotNull(bucketName); - return new OmBucketArgs(volumeName, bucketName, isVersionEnabled, - storageType, metadata, quotaInBytes, quotaInNamespace, ownerName); + OmBucketArgs omBucketArgs = + new OmBucketArgs(volumeName, bucketName, isVersionEnabled, - storageType, metadata, quotaInBytes, quotaInNamespace); ++ storageType, metadata, quotaInBytes, quotaInNamespace, ownerName); + omBucketArgs.setDefaultReplicationConfig(defaultReplicationConfig); + return omBucketArgs; } } @@@ -258,9 -253,9 +282,12 @@@ if(quotaInNamespace > 0 || quotaInNamespace == OzoneConsts.QUOTA_RESET) { builder.setQuotaInNamespace(quotaInNamespace); } + if (defaultReplicationConfig != null) { + builder.setDefaultReplicationConfig(defaultReplicationConfig.toProto()); + } + if (ownerName != null) { + builder.setOwnerName(ownerName); + } return builder.build(); } @@@ -270,18 -265,16 +297,22 @@@ * @return instance of OmBucketArgs */ public static OmBucketArgs getFromProtobuf(BucketArgs bucketArgs) { - return new OmBucketArgs(bucketArgs.getVolumeName(), - bucketArgs.getBucketName(), - bucketArgs.hasIsVersionEnabled() ? - bucketArgs.getIsVersionEnabled() : null, - bucketArgs.hasStorageType() ? StorageType.valueOf( - bucketArgs.getStorageType()) : null, - KeyValueUtil.getFromProtobuf(bucketArgs.getMetadataList()), - bucketArgs.getQuotaInBytes(), - bucketArgs.getQuotaInNamespace(), - bucketArgs.hasOwnerName() ? - bucketArgs.getOwnerName() : null); + OmBucketArgs omBucketArgs = - new OmBucketArgs(bucketArgs.getVolumeName(), bucketArgs.getBucketName(), ++ new OmBucketArgs(bucketArgs.getVolumeName(), ++ bucketArgs.getBucketName(), + bucketArgs.hasIsVersionEnabled() ? - bucketArgs.getIsVersionEnabled() : - null, bucketArgs.hasStorageType() ? - StorageType.valueOf(bucketArgs.getStorageType()) : - null, KeyValueUtil.getFromProtobuf(bucketArgs.getMetadataList()), - bucketArgs.getQuotaInBytes(), bucketArgs.getQuotaInNamespace()); ++ bucketArgs.getIsVersionEnabled() : null, ++ bucketArgs.hasStorageType() ? StorageType.valueOf( ++ bucketArgs.getStorageType()) : null, ++ KeyValueUtil.getFromProtobuf(bucketArgs.getMetadataList()), ++ bucketArgs.getQuotaInBytes(), ++ bucketArgs.getQuotaInNamespace(), ++ bucketArgs.hasOwnerName() ? ++ bucketArgs.getOwnerName() : null); + // OmBucketArgs ctor already has more arguments, so setting the default + // replication config separately. + omBucketArgs.setDefaultReplicationConfig( + new DefaultReplicationConfig(bucketArgs.getDefaultReplicationConfig())); + return omBucketArgs; } } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 2a5a69b,8bd4ea6..d1e375d --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@@ -45,12 -42,9 +45,13 @@@ import org.apache.hadoop.fs.Trash import org.apache.hadoop.fs.TrashPolicy; import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.TestDataUtil; diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index 4de0f00,2be761e..13802a1 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@@ -59,8 -59,9 +59,10 @@@ public class TestOzoneConfigurationFiel errorIfMissingXmlProps = true; xmlPropsToSkipCompare.add("hadoop.tags.custom"); xmlPropsToSkipCompare.add("ozone.om.nodes.EXAMPLEOMSERVICEID"); + xmlPropsToSkipCompare.add("ozone.om.decommissioned.nodes" + + ".EXAMPLEOMSERVICEID"); xmlPropsToSkipCompare.add("ozone.scm.nodes.EXAMPLESCMSERVICEID"); + xmlPropsToSkipCompare.add("ozone.scm.nodes.EXAMPLESCMSERVICEID"); xmlPrefixToSkipCompare.add("ipc.client.rpc-timeout.ms"); xmlPropsToSkipCompare.add("ozone.om.leader.election.minimum.timeout" + ".duration"); // Deprecated config diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index 1a18b26,f525942..19a7387 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@@ -39,14 -38,8 +39,14 @@@ import org.apache.hadoop.ozone.OFSPath import org.apache.hadoop.fs.ozone.OzoneFsShell; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; - import org.apache.hadoop.ozone.MiniOzoneOMHAClusterImpl; + import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneKeyDetails; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; +import org.apache.hadoop.ozone.client.io.KeyOutputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OzoneManager; diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 7161859,df3f2cc..b2b6a47 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -624,7 -623,7 +624,8 @@@ message BucketArgs repeated hadoop.hdds.KeyValue metadata = 7; optional uint64 quotaInBytes = 8; optional uint64 quotaInNamespace = 9; - optional hadoop.hdds.DefaultReplicationConfig defaultReplicationConfig = 10; + optional string ownerName = 10; ++ optional hadoop.hdds.DefaultReplicationConfig defaultReplicationConfig = 11; } message PrefixInfo { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 59dd3d9,900babd..64caabb --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -3610,15 -3673,6 +3676,10 @@@ public final class OzoneManager extend OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); } - public String getOMMetadataLayout() { - return configuration - .getTrimmed(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT); - } - + public ReplicationConfig getDefaultReplicationConfig() { + return ReplicationConfig.getDefault(configuration); + } + public String getOMDefaultBucketLayout() { return this.defaultBucketLayout; } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index bc70862,ea4192c..0bf7f3a --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@@ -228,9 -226,8 +228,9 @@@ public class OMAllocateBlockRequest ext omBucketInfo.incrUsedBytes(preAllocatedSpace); omResponse.setAllocateBlockResponse(AllocateBlockResponse.newBuilder() .setKeyLocation(blockLocation).build()); + OmBucketInfo shortBucketInfo = omBucketInfo.copyObject(); omClientResponse = new OMAllocateBlockResponse(omResponse.build(), - openKeyInfo, clientID, shortBucketInfo); - openKeyInfo, clientID, omBucketInfo.copyObject(), getBucketLayout()); ++ openKeyInfo, clientID, shortBucketInfo, getBucketLayout()); LOG.debug("Allocated block for Volume:{}, Bucket:{}, OpenKey:{}", volumeName, bucketName, openKeyName); diff --cc hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketCommands.java index 2731d98,2de2290..454660e --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketCommands.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/BucketCommands.java @@@ -49,7 -49,7 +49,8 @@@ import picocli.CommandLine.ParentComman GetAclBucketHandler.class, SetAclBucketHandler.class, ClearQuotaHandler.class, - SetReplicationConfigHandler.class ++ SetReplicationConfigHandler.class, + UpdateBucketHandler.class }, mixinStandardHelpOptions = true, versionProvider = HddsVersionProvider.class) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
