This is an automated email from the ASF dual-hosted git repository. umamahesh pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit aebaff55a23dcac56b84be1c2dfa0f2f1679c0fb Merge: 79f60c4 e0d4007 Author: Uma Maheswara Rao G <[email protected]> AuthorDate: Tue Feb 15 12:38:59 2022 -0800 Merge master to EC branch HDDS-3816-ec .github/workflows/post-commit.yml | 6 ++ .gitignore | 3 +- dev-support/ci/selective_ci_checks.bats | 70 ++++++++++++- dev-support/ci/selective_ci_checks.sh | 76 +++++++++++---- .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 38 ++++++++ .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 8 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 6 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 2 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 2 +- .../hdds/scm/storage/RatisBlockOutputStream.java | 2 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 2 +- .../java/org/apache/hadoop/hdds/StringUtils.java | 2 +- .../hadoop/hdds/annotation/InterfaceAudience.java | 6 +- .../org/apache/hadoop/hdds/client/OzoneQuota.java | 2 +- .../org/apache/hadoop/hdds/client/QuotaList.java | 11 ++- .../hadoop/hdds/client/ReplicationFactor.java | 2 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 2 +- .../hadoop/hdds/fs/CachingSpaceUsageSource.java | 2 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 4 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 2 +- .../org/apache/hadoop/hdds/recon/ReconConfig.java | 2 +- .../hadoop/hdds/scm/ByteStringConversion.java | 2 +- .../java/org/apache/hadoop/hdds/scm/ScmConfig.java | 2 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 +- .../scm/container/ReplicationManagerReport.java | 2 +- .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 14 +-- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 28 +++--- .../org/apache/hadoop/hdds/scm/net/NodeSchema.java | 4 +- .../hadoop/hdds/scm/net/NodeSchemaLoader.java | 8 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 8 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 8 +- .../apache/hadoop/hdds/utils/HddsVersionInfo.java | 2 +- .../hadoop/hdds/utils/ResourceSemaphore.java | 10 +- .../org/apache/hadoop/hdds/utils/UniqueId.java | 2 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 11 ++- .../hadoop/ozone/audit/AuditEventStatus.java | 2 +- .../org/apache/hadoop/ozone/audit/AuditLogger.java | 4 +- .../apache/hadoop/ozone/audit/AuditLoggerType.java | 2 +- .../org/apache/hadoop/ozone/audit/AuditMarker.java | 4 +- .../apache/hadoop/ozone/audit/AuditMessage.java | 10 +- .../hadoop/ozone/common/ChecksumByteBuffer.java | 2 +- .../ozone/common/IncrementalChunkBuffer.java | 13 +-- .../apache/hadoop/ozone/common/StorageInfo.java | 8 +- .../ozone/common/ha/ratis/RatisSnapshotInfo.java | 2 +- .../ozone/container/common/helpers/BlockData.java | 4 +- .../helpers/ContainerCommandRequestPBHelper.java | 10 +- .../java/org/apache/hadoop/ozone/lease/Lease.java | 8 +- .../hadoop/ozone/lease/LeaseCallbackExecutor.java | 2 +- .../hadoop/ozone/util/ShutdownHookManager.java | 2 +- .../common/src/main/resources/ozone-default.xml | 18 ++++ .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 4 +- .../hadoop/hdds/client/TestReplicationConfig.java | 2 +- .../java/org/apache/hadoop/hdds/fs/TestDU.java | 2 +- .../ratis/TestContainerCommandRequestMessage.java | 6 +- .../apache/hadoop/hdds/scm/ha/TestSCMNodeInfo.java | 4 +- .../hdds/scm/net/TestNetworkTopologyImpl.java | 20 ++-- .../hadoop/hdds/scm/pipeline/MockPipeline.java | 2 +- .../hadoop/hdds/utils/MockGatheringChannel.java | 2 +- .../hadoop/hdds/utils/TestResourceSemaphore.java | 6 +- .../org/apache/hadoop/ozone/audit/DummyEntity.java | 2 +- .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 4 +- .../apache/hadoop/ozone/common/TestChecksum.java | 2 +- .../hadoop/ozone/common/TestChunkBuffer.java | 20 ++-- .../hadoop/ozone/common/TestStateMachine.java | 4 +- .../ozone/container/ContainerTestHelper.java | 2 +- .../hadoop/ozone/lease/TestLeaseManager.java | 2 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 27 +----- .../container/common/helpers/ContainerMetrics.java | 6 +- .../container/common/helpers/ContainerUtils.java | 2 +- .../container/common/impl/HddsDispatcher.java | 15 +-- .../common/impl/OpenContainerBlockMap.java | 2 +- .../common/statemachine/DatanodeConfiguration.java | 2 +- .../common/statemachine/DatanodeStateMachine.java | 8 +- .../common/statemachine/StateContext.java | 12 +-- .../commandhandler/CommandDispatcher.java | 2 +- .../SetNodeOperationalStateCommandHandler.java | 11 --- .../common/states/datanode/InitDatanodeState.java | 9 +- .../states/endpoint/VersionEndpointTask.java | 2 +- .../common/transport/server/ratis/CSMMetrics.java | 2 +- .../server/ratis/ContainerStateMachine.java | 4 +- .../transport/server/ratis/XceiverServerRatis.java | 9 +- .../container/common/utils/HddsVolumeUtil.java | 2 +- .../container/common/volume/MutableVolumeSet.java | 2 +- .../container/common/volume/StorageVolume.java | 2 +- .../ozone/container/common/volume/VolumeUsage.java | 2 +- .../container/keyvalue/KeyValueContainer.java | 6 +- .../container/keyvalue/KeyValueContainerCheck.java | 6 +- .../container/keyvalue/KeyValueContainerData.java | 2 +- .../ozone/container/keyvalue/KeyValueHandler.java | 6 +- .../helpers/KeyValueContainerLocationUtil.java | 2 +- .../container/keyvalue/impl/BlockManagerImpl.java | 8 +- .../background/BlockDeletingService.java | 4 +- .../container/metadata/AbstractDatanodeStore.java | 2 +- .../container/ozoneimpl/ContainerController.java | 14 ++- .../ozoneimpl/ContainerDataScrubberMetrics.java | 6 +- .../ozoneimpl/ContainerMetadataScanner.java | 2 +- .../ContainerMetadataScrubberMetrics.java | 2 +- .../replication/GrpcReplicationClient.java | 2 +- .../container/stream/DirstreamClientHandler.java | 2 +- .../upgrade/DataNodeUpgradeFinalizer.java | 2 +- .../upgrade/VersionedDatanodeFeatures.java | 2 +- .../ozone/protocol/commands/ReregisterCommand.java | 2 +- .../hadoop/ozone/TestHddsSecureDatanodeInit.java | 2 +- .../hadoop/ozone/container/common/ScmTestMock.java | 35 ++++--- .../container/common/TestBlockDeletingService.java | 7 +- .../ozone/container/common/TestContainerCache.java | 2 +- .../container/common/TestDatanodeStateMachine.java | 5 - .../TestSchemaOneBackwardsCompatibility.java | 36 ++++--- .../container/common/helpers/TestBlockData.java | 6 +- .../common/helpers/TestDatanodeVersionFile.java | 10 +- .../container/common/impl/TestContainerSet.java | 12 +-- .../container/common/impl/TestHddsDispatcher.java | 2 +- .../container/common/interfaces/TestHandler.java | 2 +- .../common/report/TestReportPublisher.java | 2 +- .../TestCloseContainerCommandHandler.java | 4 +- .../volume/TestRoundRobinVolumeChoosingPolicy.java | 2 +- .../container/common/volume/TestVolumeSet.java | 2 +- .../keyvalue/TestKeyValueBlockIterator.java | 14 +-- .../container/keyvalue/TestKeyValueContainer.java | 2 +- .../keyvalue/TestKeyValueContainerCheck.java | 2 +- .../container/keyvalue/TestKeyValueHandler.java | 4 +- .../keyvalue/impl/CommonChunkManagerTestCases.java | 4 +- .../container/ozoneimpl/TestContainerReader.java | 8 +- .../replication/TestGrpcOutputStream.java | 8 +- .../replication/TestReplicationSupervisor.java | 2 +- .../upgrade/TestDatanodeUpgradeToScmHA.java | 6 +- hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 1 + hadoop-hdds/docs/content/security/SecuringTDE.md | 48 ++++++--- .../docs/content/security/SecuringTDE.zh.md | 4 +- .../SCMSecurityProtocolClientSideTranslatorPB.java | 2 +- ...inerLocationProtocolClientSideTranslatorPB.java | 4 +- .../scm/update/client/CRLClientUpdateHandler.java | 2 +- .../hdds/scm/update/client/ClientCRLStore.java | 4 +- .../update/client/SCMUpdateServiceGrpcClient.java | 2 +- .../x509/certificate/authority/BaseApprover.java | 2 +- .../certificate/authority/DefaultCAServer.java | 6 +- .../authority/PKIProfiles/DefaultCAProfile.java | 2 +- .../client/DefaultCertificateClient.java | 32 +++--- .../certificate/client/OMCertificateClient.java | 4 +- .../certificates/utils/CertificateSignRequest.java | 10 +- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 2 +- .../hdds/security/x509/crl/CRLInfoCodec.java | 2 +- .../hadoop/hdds/server/http/ProfileServlet.java | 2 +- .../hadoop/hdds/utils/DBCheckpointMetrics.java | 2 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 2 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 5 +- .../hadoop/hdds/utils/MetadataKeyFilters.java | 2 +- .../apache/hadoop/hdds/utils/TransactionInfo.java | 2 +- .../hadoop/hdds/utils/db/DBConfigFromFile.java | 4 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 4 +- .../apache/hadoop/hdds/utils/db/TypedTable.java | 2 +- .../hadoop/hdds/utils/db/cache/CacheKey.java | 2 +- .../hadoop/hdds/utils/db/cache/EpochEntry.java | 2 +- .../x509/certificate/authority/MockCAStore.java | 2 +- .../x509/certificate/utils/TestCRLCodec.java | 2 +- .../certificates/TestCertificateSignRequest.java | 4 +- .../x509/certificates/TestRootCertificate.java | 2 +- .../security/x509/keys/TestHDDSKeyGenerator.java | 2 +- .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 10 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 40 +------- .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 6 +- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 4 +- .../hdds/utils/db/TestTypedRDBTableStore.java | 2 +- .../hadoop/hdds/utils/db/cache/TestTableCache.java | 20 ++-- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 2 +- .../hdds/scm/container/ContainerReplicaCount.java | 18 ++-- .../hdds/scm/container/ContainerReportHandler.java | 2 +- .../hdds/scm/container/ReplicationManager.java | 24 ++--- .../balancer/AbstractFindTargetGreedy.java | 6 +- .../scm/container/balancer/ContainerBalancer.java | 2 +- .../balancer/ContainerBalancerConfiguration.java | 4 +- .../scm/container/balancer/FindSourceGreedy.java | 6 +- .../ContainerPlacementPolicyFactory.java | 2 +- .../algorithms/SCMContainerPlacementMetrics.java | 2 +- .../algorithms/SCMContainerPlacementRackAware.java | 12 +-- .../container/placement/metrics/SCMMetrics.java | 2 +- .../scm/container/states/ContainerStateMap.java | 3 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 2 +- .../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 9 +- .../apache/hadoop/hdds/scm/ha/io/CodecFactory.java | 2 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 4 +- .../hdds/scm/metadata/X509CertificateCodec.java | 2 +- .../apache/hadoop/hdds/scm/node/CommandQueue.java | 2 +- .../hdds/scm/node/NodeDecommissionManager.java | 32 +++--- .../apache/hadoop/hdds/scm/node/NodeManager.java | 2 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 2 +- .../apache/hadoop/hdds/scm/node/NodeStatus.java | 4 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 8 +- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 8 +- .../hdds/scm/node/SCMNodeStorageStatMap.java | 2 +- .../hdds/scm/node/states/Node2ObjectsMap.java | 2 +- .../hadoop/hdds/scm/node/states/NodeStateMap.java | 2 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 2 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 4 +- .../scm/pipeline/PipelineStateManagerImpl.java | 7 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +- .../scm/pipeline/WritableContainerFactory.java | 2 +- ...inerLocationProtocolServerSideTranslatorPB.java | 8 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 4 +- .../hdds/scm/safemode/DataNodeSafeModeRule.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 16 +-- .../hdds/scm/server/SCMClientProtocolServer.java | 15 +-- .../hdds/scm/server/SCMDatanodeProtocolServer.java | 6 +- .../hdds/scm/server/StorageContainerManager.java | 12 +-- .../scm/server/StorageContainerManagerStarter.java | 2 +- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 2 +- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 2 +- .../hadoop/hdds/scm/TestHddsServerUtils.java | 35 ++++++- .../hadoop/hdds/scm/block/TestBlockManager.java | 4 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 6 +- .../hdds/scm/container/SimpleMockNodeManager.java | 2 +- .../scm/container/TestContainerManagerImpl.java | 6 +- .../TestIncrementalContainerReportHandler.java | 4 +- .../container/balancer/TestContainerBalancer.java | 24 ++--- .../TestSCMContainerPlacementRackAware.java | 2 +- .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 26 ++--- .../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 2 +- .../hdds/scm/metadata/TestPipelineIDCodec.java | 2 +- .../hdds/scm/node/TestContainerPlacement.java | 2 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 4 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 12 +-- .../hadoop/hdds/scm/node/TestNodeStateManager.java | 2 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 4 +- .../hdds/scm/node/states/TestNodeStateMap.java | 4 +- .../TestPipelineDatanodesIntersection.java | 4 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 16 +-- .../scm/pipeline/TestRatisPipelineProvider.java | 4 +- .../TestOneReplicaPipelineSafeModeRule.java | 8 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 8 +- .../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +- .../server/TestSCMUpdateServiceGrpcServer.java | 24 ++--- .../scm/upgrade/TestScmStartupSlvLessThanMlv.java | 2 +- .../testutils/ReplicationNodeManagerMock.java | 2 +- .../org/apache/ozone/test/LambdaTestUtils.java | 4 +- .../scm/cli/ContainerBalancerStatusSubcommand.java | 2 +- .../cli/ReplicationManagerStatusSubcommand.java | 2 +- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 2 +- .../hdds/scm/cli/SafeModeExitSubcommand.java | 2 +- .../scm/cli/datanode/DecommissionSubCommand.java | 2 +- .../scm/cli/datanode/MaintenanceSubCommand.java | 4 +- .../scm/cli/datanode/RecommissionSubCommand.java | 2 +- .../scm/cli/container/TestReportSubCommand.java | 12 +-- .../scm/cli/datanode/TestListInfoSubcommand.java | 6 +- hadoop-ozone/client/pom.xml | 1 + .../apache/hadoop/ozone/client/ObjectStore.java | 6 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 12 +-- .../hadoop/ozone/client/OzoneClientFactory.java | 2 +- .../org/apache/hadoop/ozone/client/OzoneKey.java | 2 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 2 +- .../client/checksum/BaseFileChecksumHelper.java | 9 +- .../checksum/ReplicatedFileChecksumHelper.java | 6 +- .../ozone/client/io/BlockOutputStreamEntry.java | 16 ++- .../hadoop/ozone/client/rpc/OzoneKMSUtil.java | 4 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 24 ++--- .../hadoop/ozone/client/TestHddsClientUtils.java | 4 +- .../checksum/TestReplicatedFileChecksumHelper.java | 4 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 4 +- .../java/org/apache/hadoop/ozone/OzoneAcl.java | 8 +- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 6 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 10 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 1 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 2 +- .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 2 +- .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 4 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 4 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 2 +- .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 6 +- .../hadoop/ozone/om/helpers/ServiceInfo.java | 2 +- .../hadoop/ozone/om/helpers/WithObjectID.java | 2 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 8 +- .../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 14 +-- .../hadoop/ozone/security/acl/OzoneObjInfo.java | 2 +- .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 2 +- .../org/apache/hadoop/ozone/util/RadixTree.java | 2 +- .../ozone/om/ha/TestOMFailoverProxyProvider.java | 2 +- .../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 2 +- .../ozone/security/TestGDPRSymmetricKey.java | 2 +- .../ozone/security/acl/TestOzoneObjInfo.java | 4 +- .../apache/hadoop/ozone/util/TestRadixTree.java | 4 +- hadoop-ozone/dev-support/checks/bats.sh | 8 +- hadoop-ozone/dev-support/checks/checkstyle.sh | 3 + hadoop-ozone/dist/pom.xml | 2 +- .../dist/src/main/compose/compatibility/.env | 1 + .../main/compose/compatibility/docker-compose.yaml | 2 +- hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 + .../src/main/compose/ozone-csi/docker-compose.yaml | 8 +- hadoop-ozone/dist/src/main/compose/ozone-ha/.env | 1 + .../src/main/compose/ozone-ha/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone-mr/hadoop27/.env | 1 + .../compose/ozone-mr/hadoop27/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 + .../compose/ozone-mr/hadoop31/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 + .../compose/ozone-mr/hadoop32/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-mr/hadoop33/.env | 1 + .../compose/ozone-mr/hadoop33/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozone-om-ha/.env | 1 + .../dist/src/main/compose/ozone-om-ha/Dockerfile | 3 +- .../dist/src/main/compose/ozone-om-prepare/.env | 6 +- .../compose/ozone-om-prepare/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone-topology/.env | 1 + .../compose/ozone-topology/docker-compose.yaml | 16 +-- hadoop-ozone/dist/src/main/compose/ozone/.env | 1 + .../src/main/compose/ozone/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone/freon-ockg.yaml | 2 +- .../dist/src/main/compose/ozone/freon-rk.yaml | 2 +- hadoop-ozone/dist/src/main/compose/ozone/test.sh | 6 -- .../dist/src/main/compose/ozoneblockade/.env | 1 + .../main/compose/ozoneblockade/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozones3-haproxy/.env | 1 + .../compose/ozones3-haproxy/docker-compose.yaml | 12 +-- .../dist/src/main/compose/ozonescripts/.env | 1 + .../dist/src/main/compose/ozonescripts/Dockerfile | 3 +- .../main/compose/ozonescripts/docker-compose.yaml | 3 + .../dist/src/main/compose/ozonesecure-ha/.env | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 22 ++--- .../dist/src/main/compose/ozonesecure-mr/.env | 1 + .../compose/ozonesecure-mr/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozonesecure-mr/test.sh | 3 - .../dist/src/main/compose/ozonesecure/.env | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 10 +- hadoop-ozone/dist/src/main/compose/restart/.env | 2 +- .../src/main/compose/restart/docker-compose.yaml | 2 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 3 +- .../dist/src/main/compose/upgrade/compose/ha/.env | 1 + .../main/compose/upgrade/compose/ha/docker-config | 4 +- .../src/main/compose/upgrade/compose/non-ha/.env | 1 + hadoop-ozone/dist/src/main/compose/xcompat/.env | 1 + .../dist/src/main/compose/xcompat/clients.yaml | 2 +- .../dist/src/main/compose/xcompat/new-cluster.yaml | 2 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 3 +- .../src/main/smoketest/compatibility/read.robot | 9 +- .../compatibility/{write.robot => setup.robot} | 16 ++- .../src/main/smoketest/compatibility/write.robot | 7 +- .../dist/src/main/smoketest/mapreduce.robot | 1 + .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 1 + hadoop-ozone/dist/src/shell/upgrade/1.0.0.sh | 3 +- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 10 +- .../hadoop/ozone/MiniOzoneLoadGenerator.java | 2 +- .../hadoop/ozone/loadgenerators/LoadBucket.java | 2 +- .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 14 +-- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 2 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 ++++---- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 4 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 34 +++---- .../fs/ozone/contract/ITestOzoneContractUtils.java | 4 +- .../hadoop/fs/ozone/contract/OzoneContract.java | 4 +- .../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 2 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 2 +- .../TestRatisPipelineCreateAndDestroy.java | 2 +- .../safemode/TestSCMSafeModeWithPipelineRules.java | 6 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 6 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +- .../hadoop/ozone/MiniOzoneClusterProvider.java | 4 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 4 +- .../org/apache/hadoop/ozone/OzoneTestUtils.java | 2 +- .../ozone/TestContainerBalancerOperations.java | 4 +- .../hadoop/ozone/TestContainerOperations.java | 2 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 2 +- .../ozone/client/CertificateClientTestImpl.java | 4 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 2 +- .../client/rpc/TestDiscardPreallocatedBlocks.java | 2 +- .../client/rpc/TestFailureHandlingByClient.java | 2 +- .../client/rpc/TestOzoneAtRestEncryption.java | 10 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 10 +- .../rpc/TestOzoneClientRetriesOnExceptions.java | 4 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 60 ++++++------ .../rpc/TestOzoneRpcClientForAclAuditLog.java | 18 ++-- .../TestOzoneRpcClientWithKeyLatestVersion.java | 2 +- .../hadoop/ozone/client/rpc/TestReadRetries.java | 2 +- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 6 +- .../ozone/client/rpc/TestWatchForCommit.java | 2 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 2 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 8 +- .../TestCloseContainerByPipeline.java | 2 +- .../commandhandler/TestDeleteContainerHandler.java | 2 +- .../container/metrics/TestContainerMetrics.java | 4 +- .../container/ozoneimpl/TestOzoneContainer.java | 2 +- .../container/server/TestContainerServer.java | 2 +- .../server/TestSecureContainerServer.java | 6 +- .../TestDatanodeHddsVolumeFailureDetection.java | 2 +- .../ozone/freon/TestHadoopNestedDirGenerator.java | 26 ++--- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 14 +-- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 2 +- .../ozone/om/TestOmStartupSlvLessThanMlv.java | 2 +- .../ozone/om/TestOzoneManagerConfiguration.java | 4 +- .../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 8 +- .../ozone/om/TestOzoneManagerHAWithData.java | 2 +- .../ozone/om/TestOzoneManagerRestInterface.java | 2 +- .../hadoop/ozone/recon/TestReconScmSnapshot.java | 2 +- .../ozone/recon/TestReconWithOzoneManager.java | 11 ++- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 8 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 2 +- .../hadoop/ozone/scm/TestCloseContainer.java | 4 +- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 6 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 10 +- .../scm/node/TestDecommissionAndMaintenance.java | 18 ++-- .../hadoop/ozone/scm/node/TestQueryNode.java | 2 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 22 +++-- .../hadoop/ozone/om/helpers/TestOmPrefixInfo.java | 2 +- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 58 +++++------ .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 8 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 22 ++--- .../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 2 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +-- .../hadoop/ozone/om/OzoneManagerPrepareState.java | 4 +- .../hadoop/ozone/om/OzoneManagerStarter.java | 2 +- .../hadoop/ozone/om/S3SecretManagerImpl.java | 2 +- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 12 +-- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +-- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 4 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 6 +- .../hadoop/ozone/om/request/OMClientRequest.java | 6 +- .../om/request/bucket/OMBucketCreateRequest.java | 6 +- .../request/bucket/OMBucketSetPropertyRequest.java | 6 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 2 +- .../om/request/file/OMDirectoryCreateRequest.java | 2 +- .../ozone/om/request/file/OMFileCreateRequest.java | 2 +- .../ozone/om/request/file/OMFileRequest.java | 10 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 4 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 2 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 2 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 8 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 3 +- .../multipart/S3MultipartUploadAbortRequest.java | 2 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 6 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 2 +- .../request/volume/acl/OMVolumeAddAclRequest.java | 2 +- .../volume/acl/OMVolumeRemoveAclRequest.java | 2 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 2 +- .../OzoneDelegationTokenSecretManager.java | 2 +- .../hadoop/ozone/security/OzoneSecretStore.java | 4 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 2 +- .../apache/hadoop/ozone/om/TestOMDBDefinition.java | 2 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 14 +-- .../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 2 +- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 4 +- .../om/ratis/TestOzoneManagerRatisServer.java | 2 +- .../om/ratis/TestOzoneManagerStateMachine.java | 2 +- .../request/file/TestOMDirectoryCreateRequest.java | 2 +- .../om/request/file/TestOMFileCreateRequest.java | 6 +- .../om/request/key/TestOMKeyCommitRequest.java | 4 +- .../request/key/TestOMKeyCommitRequestWithFSO.java | 2 +- .../om/request/key/TestOMKeyCreateRequest.java | 2 +- .../request/key/TestOMKeyDeleteRequestWithFSO.java | 4 +- .../key/TestOMKeyPurgeRequestAndResponse.java | 2 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 2 +- .../TestS3MultipartUploadCompleteRequest.java | 2 +- .../request/volume/TestOMVolumeCreateRequest.java | 2 +- .../volume/TestOMVolumeSetQuotaRequest.java | 2 +- .../ozone/om/response/TestCleanupTableInfo.java | 4 +- .../ozone/om/upgrade/TestOMUpgradeFinalizer.java | 2 +- .../om/upgrade/TestOzoneManagerPrepareState.java | 2 +- .../TestOzoneDelegationTokenSecretManager.java | 6 +- .../ozone/security/TestOzoneTokenIdentifier.java | 6 +- .../security/acl/TestOzoneNativeAuthorizer.java | 6 +- .../hadoop/ozone/security/acl/TestVolumeOwner.java | 4 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 16 ++- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 9 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 18 ++++ .../fs/ozone/BasicRootedOzoneFileSystem.java | 3 +- .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 + .../apache/hadoop/fs/ozone/OzoneClientUtils.java | 27 +++++- .../apache/hadoop/fs/ozone/OzoneFSInputStream.java | 2 +- .../hadoop/fs/ozone/TestOzoneClientUtils.java | 35 +++++++ hadoop-ozone/ozonefs-shaded/pom.xml | 2 +- .../org/hadoop/ozone/recon/codegen/SqlDbUtils.java | 4 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 7 ++ .../ozone/recon/api/MetricsProxyEndpoint.java | 4 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 2 +- .../hadoop/ozone/recon/api/PipelineEndpoint.java | 2 +- .../hadoop/ozone/recon/codec/NSSummaryCodec.java | 2 +- .../ozone/recon/fsck/ContainerHealthTask.java | 4 +- .../ozone/recon/scm/ReconContainerManager.java | 4 +- .../ozone/recon/scm/ReconPipelineFactory.java | 2 +- .../scm/ReconStorageContainerManagerFacade.java | 4 +- .../recon/spi/impl/ContainerKeyPrefixCodec.java | 2 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 62 ++++++++++-- .../ozone/recon/tasks/FileSizeCountTask.java | 4 +- .../hadoop/ozone/recon/tasks/TableCountTask.java | 2 +- .../ozone/recon/api/TestTaskStatusService.java | 2 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 2 +- .../TestUtilizationSchemaDefinition.java | 2 +- .../impl/TestOzoneManagerServiceProviderImpl.java | 88 ++++++++++++++++- .../recon/tasks/TestContainerKeyMapperTask.java | 2 +- .../ozone/recon/tasks/TestOMDBUpdatesHandler.java | 4 +- .../ozone/recon/tasks/TestTableCountTask.java | 2 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 17 ++-- .../hadoop/ozone/s3/VirtualHostStyleFilter.java | 8 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 58 +++++------ .../hadoop/ozone/s3/endpoint/EndpointBase.java | 18 ++-- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 108 +++++++++------------ .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 6 +- .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 4 +- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 20 +++- .../hadoop/ozone/s3/signature/Credential.java | 2 +- .../apache/hadoop/ozone/s3/util/ContinueToken.java | 2 +- .../hadoop/ozone/s3/TestOzoneClientProducer.java | 39 +++++++- .../ozone/s3/TestVirtualHostStyleFilter.java | 2 +- .../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 2 +- .../ozone/admin/om/FinalizeUpgradeSubCommand.java | 6 +- .../admin/scm/FinalizeScmUpgradeSubcommand.java | 6 +- .../admin/scm/FinalizeUpgradeCommandUtil.java | 4 +- .../hadoop/ozone/audit/parser/AuditParser.java | 2 +- .../ozone/audit/parser/common/DatabaseHelper.java | 16 +-- .../parser/handler/TemplateCommandHandler.java | 2 +- .../ozone/audit/parser/model/AuditEntry.java | 22 ++--- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 6 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 8 +- .../apache/hadoop/ozone/debug/PrefixParser.java | 2 +- .../apache/hadoop/ozone/debug/ReadReplicas.java | 14 +-- .../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 12 +-- .../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 4 +- .../hadoop/ozone/freon/HadoopFsGenerator.java | 2 +- .../hadoop/ozone/freon/StreamingGenerator.java | 2 +- .../GenerateOzoneRequiredConfigurations.java | 2 +- .../apache/hadoop/ozone/shell/OzoneAddress.java | 4 +- .../ozone/shell/bucket/CreateBucketHandler.java | 2 +- .../hadoop/ozone/audit/parser/TestAuditParser.java | 2 +- .../hadoop/ozone/conf/TestGetConfOptions.java | 4 +- .../TestGenerateOzoneRequiredConfigurations.java | 4 +- .../org/apache/hadoop/test/OzoneTestDriver.java | 6 +- pom.xml | 26 ++++- 534 files changed, 2072 insertions(+), 1569 deletions(-) diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java index 0d59e8f,064ce6e..351f69d --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java @@@ -124,21 -136,20 +136,35 @@@ public class OzoneClientConfig tags = ConfigTag.CLIENT) private boolean checksumVerify = true; + @Config(key = "max.ec.stripe.write.retries", + defaultValue = "10", + description = "Ozone EC client to retry stripe to new block group on" + + " failures.", + tags = ConfigTag.CLIENT) + private int maxECStripeWriteRetries = 10; + + @Config(key = "exclude.nodes.expiry.time", + defaultValue = "600000", + description = "Time after which an excluded node is reconsidered for" + + " writes in EC. If the value is zero, the node is excluded for the" + + " life of the client", + tags = ConfigTag.CLIENT) + private long excludeNodesExpiryTime = 10 * 60 * 1000; + + @Config(key = "checksum.combine.mode", + defaultValue = "COMPOSITE_CRC", + description = "The combined checksum type [MD5MD5CRC / COMPOSITE_CRC] " + + "determines which algorithm would be used to compute file checksum." + + "COMPOSITE_CRC calculates the combined CRC of the whole file, " + + "where the lower-level chunk/block checksums are combined into " + + "file-level checksum." + + "MD5MD5CRC calculates the MD5 of MD5 of checksums of individual " + + "chunks." + + "Default checksum type is COMPOSITE_CRC.", + tags = ConfigTag.CLIENT) + private String checksumCombineMode = + ChecksumCombineMode.COMPOSITE_CRC.name(); + @PostConstruct private void validate() { Preconditions.checkState(streamBufferSize > 0); diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index 2d14d4e,f5c0b62..116e767 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@@ -462,9 -414,8 +462,9 @@@ public final class Pipeline private UUID leaderId = null; private Instant creationTimestamp = null; private UUID suggestedLeaderId = null; + private Map<DatanodeDetails, Integer> replicaIndexes = new HashMap<>(); - public Builder() {} + public Builder() { } public Builder(Pipeline pipeline) { this.id = pipeline.id; diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java index 19cc13e,1315ad5..4811187 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java @@@ -277,9 -166,8 +277,9 @@@ public class TestReplicationConfig */ @Test public void testValidationBasedOnConfig() { + assumeRatisOrStandaloneType(); OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OZONE_REPLICATION+".allowed-configs", + conf.set(OZONE_REPLICATION + ".allowed-configs", "^STANDALONE/ONE|RATIS/THREE$"); conf.set(OZONE_REPLICATION, factor); conf.set(OZONE_REPLICATION_TYPE, type); diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java index 4932d5a,9291d33..b23da41 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketArgs.java @@@ -270,21 -241,18 +270,21 @@@ public final class OmBucketArgs extend BucketArgs.Builder builder = BucketArgs.newBuilder(); builder.setVolumeName(volumeName) .setBucketName(bucketName); - if(isVersionEnabled != null) { + if (isVersionEnabled != null) { builder.setIsVersionEnabled(isVersionEnabled); } - if(storageType != null) { + if (storageType != null) { builder.setStorageType(storageType.toProto()); } - if(quotaInBytes > 0 || quotaInBytes == OzoneConsts.QUOTA_RESET) { + if (quotaInBytes > 0 || quotaInBytes == OzoneConsts.QUOTA_RESET) { builder.setQuotaInBytes(quotaInBytes); } - if(quotaInNamespace > 0 || quotaInNamespace == OzoneConsts.QUOTA_RESET) { + if (quotaInNamespace > 0 || quotaInNamespace == OzoneConsts.QUOTA_RESET) { builder.setQuotaInNamespace(quotaInNamespace); } + if (defaultReplicationConfig != null) { + builder.setDefaultReplicationConfig(defaultReplicationConfig.toProto()); + } if (ownerName != null) { builder.setOwnerName(ownerName); } diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java index c943341,51c7d54..84a1cc2 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/protocolPB/OMPBHelper.java @@@ -146,57 -141,8 +146,57 @@@ public final class OMPBHelper ezKeyVersionName); } + public static DefaultReplicationConfig convert( + HddsProtos.DefaultReplicationConfig defaultReplicationConfig) { + if (defaultReplicationConfig == null) { + throw new IllegalArgumentException( + "Invalid argument: default replication config" + " is null"); + } + + final ReplicationType type = + ReplicationType.fromProto(defaultReplicationConfig.getType()); + DefaultReplicationConfig defaultReplicationConfigObj = null; + switch (type) { + case EC: + defaultReplicationConfigObj = new DefaultReplicationConfig(type, + new ECReplicationConfig( + defaultReplicationConfig.getEcReplicationConfig())); + break; + default: + final ReplicationFactor factor = + ReplicationFactor.fromProto(defaultReplicationConfig.getFactor()); + defaultReplicationConfigObj = new DefaultReplicationConfig(type, factor); + } + return defaultReplicationConfigObj; + } + + public static HddsProtos.DefaultReplicationConfig convert( + DefaultReplicationConfig defaultReplicationConfig) { + if (defaultReplicationConfig == null) { + throw new IllegalArgumentException( + "Invalid argument: default replication config" + " is null"); + } + + final HddsProtos.DefaultReplicationConfig.Builder builder = + HddsProtos.DefaultReplicationConfig.newBuilder(); + builder.setType( + ReplicationType.toProto(defaultReplicationConfig.getType())); + + if (defaultReplicationConfig.getFactor() != null) { + builder.setFactor(ReplicationFactor + .toProto(defaultReplicationConfig.getFactor())); + } + + if (defaultReplicationConfig.getEcReplicationConfig() != null) { + builder.setEcReplicationConfig( + defaultReplicationConfig.getEcReplicationConfig().toProto()); + } + + return builder.build(); + } + public static CipherSuite convert(CipherSuiteProto proto) { - switch(proto) { + switch (proto) { case AES_CTR_NOPADDING: return CipherSuite.AES_CTR_NOPADDING; default: diff --cc hadoop-ozone/dist/src/main/compose/ozone/test.sh index 070cebf,59a90ed..665a3c9 --- a/hadoop-ozone/dist/src/main/compose/ozone/test.sh +++ b/hadoop-ozone/dist/src/main/compose/ozone/test.sh @@@ -53,14 -53,6 +53,8 @@@ execute_robot_test scm admincl execute_robot_test scm -v SCHEME:ofs -v BUCKET_TYPE:link -N ozonefs-fso-ofs-link ozonefs/ozonefs.robot execute_robot_test scm -v SCHEME:o3fs -v BUCKET_TYPE:bucket -N ozonefs-fso-o3fs-bucket ozonefs/ozonefs.robot - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-fso-layout-objectputget s3/objectputget.robot - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-fso-layout-objectdelete s3/objectdelete.robot - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-fso-layout-objectcopy s3/objectcopy.robot - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-fso-layout-objectmultidelete s3/objectmultidelete.robot - execute_robot_test scm -v BUCKET:${bucket} -N s3-${bucket}-fso-layout-MultipartUpload s3/MultipartUpload.robot - +execute_robot_test scm ec + stop_docker_env generate_report diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java index b632aae,92333b2..f218160 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java @@@ -16,15 -16,16 +16,22 @@@ */ package org.apache.hadoop.fs.ozone; + import com.google.common.base.Preconditions; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.fs.FileChecksum; + import org.apache.hadoop.hdds.scm.OzoneClientConfig; + import org.apache.hadoop.ozone.client.checksum.BaseFileChecksumHelper; import org.apache.hadoop.ozone.client.ObjectStore; import org.apache.hadoop.ozone.client.OzoneBucket; + import org.apache.hadoop.ozone.client.OzoneVolume; + import org.apache.hadoop.ozone.client.checksum.ReplicatedFileChecksumHelper; + import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@@ -74,119 -72,18 +81,133 @@@ public final class OzoneClientUtils return bucket.getBucketLayout(); } + /** + * This API used to resolve the client side configuration preference for file + * system layer implementations. + * + * @param replication - replication value passed from FS API. + * @param clientConfiguredReplConfig - Client side configured replication + * config. + * @param bucketReplConfig - server side bucket default replication + * config. + * @param config - Ozone configuration object. + * @return client resolved replication config. + */ + public static ReplicationConfig resolveClientSideReplicationConfig( + short replication, ReplicationConfig clientConfiguredReplConfig, + ReplicationConfig bucketReplConfig, OzoneConfiguration config) { + ReplicationConfig clientDeterminedReplConfig = null; + + boolean isECBucket = bucketReplConfig != null && bucketReplConfig + .getReplicationType() == HddsProtos.ReplicationType.EC; + + // if bucket replication config configured with EC, we will give high + // preference to server side bucket defaults. + // Why we give high prefernce to EC is, there is no way for file system + // interfaces to pass EC replication. So, if one configures EC at bucket, + // we consider EC to take preference. in short, keys created from file + // system under EC bucket will always be EC'd. + if (isECBucket) { + // if bucket is EC, don't bother client provided configs, let's pass + // bucket config. + clientDeterminedReplConfig = bucketReplConfig; + } else { + // Let's validate the client side available replication configs. + boolean isReplicationInSupportedList = + (replication == ReplicationFactor.ONE + .getValue() || replication == ReplicationFactor.THREE.getValue()); + if (isReplicationInSupportedList) { + if (clientConfiguredReplConfig != null) { + // Uses the replication(short value) passed from file system API and + // construct replication config object. + // In case if client explicitely configured EC in configurations, we + // always take EC as priority as EC replication can't be expressed in + // filesystem API. + clientDeterminedReplConfig = ReplicationConfig + .adjustReplication(clientConfiguredReplConfig, replication, + config); + } else { + // In file system layers, replication parameter always passed. + // so, to respect the API provided replication value, we take RATIS as + // default type. + clientDeterminedReplConfig = ReplicationConfig + .parse(ReplicationType.RATIS, Short.toString(replication), + config); + } + } else { + // API passed replication number is not in supported replication list. + // So, let's use whatever available in client side configured. + // By default it will be null, so server will use server defaults. + clientDeterminedReplConfig = clientConfiguredReplConfig; + } + } + return clientDeterminedReplConfig; + } + + static ReplicationConfig getClientConfiguredReplicationConfig( + ConfigurationSource config) { + String replication = config.get(OZONE_REPLICATION); + if (replication == null) { + return null; + } + return ReplicationConfig.parse(ReplicationType.valueOf( + config.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT)), + replication, config); + } + + /** + * Gets the client side replication config by checking user passed values vs + * client configured values. + * @param userPassedType - User provided replication type. + * @param userPassedReplication - User provided replication. + * @param clientSideConfig - Client side configuration. + * @return ReplicationConfig. + */ + public static ReplicationConfig validateAndGetClientReplicationConfig( + ReplicationType userPassedType, String userPassedReplication, + OzoneConfiguration clientSideConfig) { + // Priority 1: User passed replication config values. + // Priority 2: Client side configured replication config values. + /* if above two are not available, we should just return null and clients + can pass null replication config to server. Now server will take the + decision of finding the replication config( either from bucket defaults + or server defaults). */ + ReplicationType clientReplicationType = userPassedType; + String clientReplication = userPassedReplication; + String clientConfiguredDefaultType = + clientSideConfig.get(OZONE_REPLICATION_TYPE); + if (userPassedType == null && clientConfiguredDefaultType != null) { + clientReplicationType = + ReplicationType.valueOf(clientConfiguredDefaultType); + } + + String clientConfiguredDefaultReplication = + clientSideConfig.get(OZONE_REPLICATION); - if (userPassedReplication == null - && clientConfiguredDefaultReplication != null) { ++ if (userPassedReplication == null && clientConfiguredDefaultReplication != null) { + clientReplication = clientConfiguredDefaultReplication; + } + + // if clientReplicationType or clientReplication is null, then we just pass + // replication config as null, so that server will take decision. + if (clientReplicationType == null || clientReplication == null) { + return null; + } + return ReplicationConfig + .parse(clientReplicationType, clientReplication, clientSideConfig); + } ++ + public static FileChecksum getFileChecksumWithCombineMode(OzoneVolume volume, + OzoneBucket bucket, String keyName, long length, + OzoneClientConfig.ChecksumCombineMode combineMode, + ClientProtocol rpcClient) throws IOException { + Preconditions.checkArgument(length >= 0); + + if (keyName.length() == 0) { + return null; + } + BaseFileChecksumHelper helper = new ReplicatedFileChecksumHelper( + volume, bucket, keyName, length, rpcClient); + helper.compute(); + return helper.getFileChecksum(); + } } diff --cc hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java index f610b75,a9d4f98..f977577 --- a/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java +++ b/hadoop-ozone/ozonefs-common/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneClientUtils.java @@@ -16,211 -16,44 +16,246 @@@ */ package org.apache.hadoop.fs.ozone; + import org.apache.hadoop.fs.FileChecksum; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.ozone.OzoneConfigKeys; + import org.apache.hadoop.ozone.client.OzoneBucket; + import org.apache.hadoop.ozone.client.OzoneVolume; + import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.junit.Assert; import org.junit.Test; + import java.io.IOException; + + import static org.junit.Assert.assertNull; + import static org.mockito.Mockito.mock; + /** - * Unit tests for OzoneClientUtils. + * Tests the behavior of OzoneClientUtils APIs. */ public class TestOzoneClientUtils { + private ReplicationConfig ecReplicationConfig = + new ECReplicationConfig("rs-3-2-1024K"); + private ReplicationConfig ratis3ReplicationConfig = + new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE); + private ReplicationConfig ratis1ReplicationConfig = + new RatisReplicationConfig(HddsProtos.ReplicationFactor.ONE); + + @Test(expected = IllegalArgumentException.class) + public void testNegativeLength() throws IOException { + OzoneVolume volume = mock(OzoneVolume.class); + OzoneBucket bucket = mock(OzoneBucket.class); + String keyName = "dummy"; + ClientProtocol clientProtocol = mock(ClientProtocol.class); + OzoneClientUtils.getFileChecksumWithCombineMode(volume, bucket, keyName, + -1, OzoneClientConfig.ChecksumCombineMode.MD5MD5CRC, clientProtocol); + + } + + @Test + public void testEmptyKeyName() throws IOException { + OzoneVolume volume = mock(OzoneVolume.class); + OzoneBucket bucket = mock(OzoneBucket.class); + String keyName = ""; + ClientProtocol clientProtocol = mock(ClientProtocol.class); + FileChecksum checksum = + OzoneClientUtils.getFileChecksumWithCombineMode(volume, bucket, keyName, + 1, OzoneClientConfig.ChecksumCombineMode.MD5MD5CRC, + clientProtocol); + + assertNull(checksum); + } -} ++ + @Test + public void testResolveClientSideRepConfigWhenBucketHasEC() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) 3, null, + ecReplicationConfig, new OzoneConfiguration()); + // Bucket default is EC. + Assert.assertEquals(ecReplicationConfig, replicationConfig); + } + + /** + * When bucket replication is null and it should respect fs passed value. + */ + @Test + public void testResolveClientSideRepConfigWhenBucketHasNull() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) 3, null, null, + new OzoneConfiguration()); + // Passed replication is 3 - Ozone mapped replication is ratis THREE + Assert.assertEquals(ratis3ReplicationConfig, replicationConfig); + } + + /** + * When bucket replication is null and it should return null if fs passed + * value is invalid. + */ + @Test + public void testResolveClientSideRepConfigWhenFSPassedReplicationIsInvalid() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) -1, null, null, + new OzoneConfiguration()); + // client configured value also null. + // This API caller should leave the decision to server. + Assert.assertNull(replicationConfig); + } + + /** + * When bucket default is non-EC and client side values are not valid, we + * would just return null, so servers can make decision in this case. + */ + @Test + public void testResolveRepConfWhenFSPassedIsInvalidButBucketDefaultNonEC() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) -1, null, ratis3ReplicationConfig, + new OzoneConfiguration()); + // Configured client config also null. + Assert.assertNull(replicationConfig); + } + + /** + * When bucket default is non-EC and client side value is valid, we + * would should return client side valid value. + */ + @Test + public void testResolveRepConfWhenFSPassedIsValidButBucketDefaultNonEC() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) 1, null, ratis3ReplicationConfig, + new OzoneConfiguration()); + // Passed value is replication one - Ozone mapped value is ratis ONE + Assert.assertEquals(ratis1ReplicationConfig, replicationConfig); + } + + /** + * When bucket default is EC and client side value also valid, we would just + * return bucket default EC. + */ + @Test + public void testResolveRepConfWhenFSPassedIsValidButBucketDefaultEC() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) 3, ratis3ReplicationConfig, + ecReplicationConfig, new OzoneConfiguration()); + // Bucket default is EC + Assert.assertEquals(ecReplicationConfig, replicationConfig); + } + + /** + * When bucket default is non-EC and client side passed value also not valid + * but configured value is valid, we would just return configured value. + */ + @Test + public void testResolveRepConfWhenFSPassedIsInvalidAndBucketDefaultNonEC() { + ReplicationConfig replicationConfig = OzoneClientUtils + .resolveClientSideReplicationConfig( + (short) -1, ratis3ReplicationConfig, ratis1ReplicationConfig, + new OzoneConfiguration()); + // Configured value is ratis THREE + Assert.assertEquals(ratis3ReplicationConfig, replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user passed valid config + * values. + */ + @Test + public void testValidateAndGetRepConfWhenValidUserPassedValues() { + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(ReplicationType.RATIS, "1", + new OzoneConfiguration()); + // Configured value is ratis ONE + Assert.assertEquals(ratis1ReplicationConfig, replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user passed null values. + */ + @Test + public void testValidateAndGetRepConfWhenValidUserPassedNullValues() { + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(null, null, + new OzoneConfiguration()); + Assert.assertNull(replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user passed null values + * but client config has valid values. + */ + @Test + public void testValidateAndGetRepConfWhenValidConfigValues() { + OzoneConfiguration clientSideConfig = new OzoneConfiguration(); + clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, "EC"); + clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024K"); + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(null, null, clientSideConfig); + Assert.assertEquals(ecReplicationConfig, replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user passed null values + * but client config has valid values. + */ + @Test + public void testValidateAndGetRepConfWhenNullTypeFromUser() { + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(null, "3", + new OzoneConfiguration()); + Assert.assertNull(replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user passed null + * replication but valid type. + */ + @Test + public void testValidateAndGetRepConfWhenNullReplicationFromUser() { + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(ReplicationType.EC, null, + new OzoneConfiguration()); + Assert.assertNull(replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user pass null values but + * config has only replication configured. + */ + @Test + public void testValidateAndGetRepConfWhenNullTypeConfigValues() { + OzoneConfiguration clientSideConfig = new OzoneConfiguration(); + clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION, "rs-3-2-1024K"); + //By default config values are null. Let's don't set type to keep it as + // null. + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(null, null, clientSideConfig); + Assert.assertNull(replicationConfig); + } + + /** + * Tests validateAndGetClientReplicationConfig with user pass null values but + * config has only type configured. + */ + @Test + public void testValidateAndGetRepConfWhenNullReplicationConfigValues() { + OzoneConfiguration clientSideConfig = new OzoneConfiguration(); + clientSideConfig.set(OzoneConfigKeys.OZONE_REPLICATION_TYPE, "EC"); + //By default config values are null. Let's don't set replication to keep it + // as null. + ReplicationConfig replicationConfig = OzoneClientUtils + .validateAndGetClientReplicationConfig(null, null, clientSideConfig); + Assert.assertNull(replicationConfig); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
