This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-4944 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 9787f7016457f2171f9ba938e288867f101148bf Merge: c826ec2 3accc98 Author: Siyao Meng <[email protected]> AuthorDate: Wed Aug 18 14:45:43 2021 -0700 Merge remote-tracking branch 'asf/master' into HDDS-4944 Conflicts: hadoop-ozone/dist/src/main/license/jar-report.txt Change-Id: I6c4500bb49fafa3ed27a78233b8e7b23ad5a6fd4 .github/workflows/post-commit.yml | 130 ++-- SECURITY.md | 1 + dev-support/ci/lib/_all_libs.sh | 34 + dev-support/ci/lib/_initialization.sh | 140 ++++ dev-support/ci/lib/_script_init.sh | 42 ++ dev-support/ci/lib/_start_end.sh | 112 +++ dev-support/ci/lib/_traps.sh | 38 + dev-support/ci/lib/_verbosity.sh | 57 ++ dev-support/ci/selective_ci_checks.bats | 210 ++++++ dev-support/ci/selective_ci_checks.sh | 522 +++++++++++++ .../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 10 + .../hadoop/hdds/client/ReplicationConfig.java | 10 +- .../hdds/client/ReplicationConfigValidator.java | 63 ++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++ .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 -------- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +- .../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 + .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 + .../hadoop/hdds/scm/net/NetworkTopology.java | 7 + .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 + .../protocol/StorageContainerLocationProtocol.java | 11 +- .../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +- .../common/src/main/resources/ozone-default.xml | 142 +++- .../hadoop/hdds/client/TestReplicationConfig.java | 24 + .../client/TestReplicationConfigValidator.java | 73 ++ .../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +- .../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 + .../common/statemachine/DatanodeConfiguration.java | 48 ++ .../common/statemachine/DatanodeStateMachine.java | 11 +- .../commandhandler/DeleteBlocksCommandHandler.java | 274 +++++-- .../server/ratis/ContainerStateMachine.java | 8 +- .../transport/server/ratis/XceiverServerRatis.java | 18 +- .../ozone/container/keyvalue/KeyValueHandler.java | 19 +- .../OnDemandContainerReplicationSource.java | 10 +- .../container/stream/DirectoryServerSource.java | 12 +- .../upgrade/DataNodeUpgradeFinalizer.java | 4 +- .../docs/content/feature/Nonrolling-Upgrade.md | 107 +++ hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +- hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/_index.zh.md | 14 +- .../feature/how-to-do-a-nonrolling-upgrade.md | 54 -- hadoop-hdds/docs/content/interface/Ofs.md | 28 +- hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++ .../container/common/helpers/MoveDataNodePair.java | 71 ++ .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 + ...inerLocationProtocolClientSideTranslatorPB.java | 30 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 2 + .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +- .../src/main/proto/ScmAdminProtocol.proto | 4 +- .../interface-client/src/main/proto/hdds.proto | 5 + .../src/main/proto/SCMRatisProtocol.proto | 1 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 47 +- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 27 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 281 ++++--- .../hdds/scm/block/DeletedBlockLogImplV2.java | 432 ----------- .../hdds/scm/block/SCMBlockDeletingService.java | 16 +- .../scm/block/ScmBlockDeletingServiceMetrics.java | 186 +++++ .../scm/command/CommandStatusReportHandler.java | 6 +- .../hdds/scm/container/ContainerReplicaCount.java | 1 + .../hdds/scm/container/ContainerStateManager.java | 10 +- .../hdds/scm/container/ReplicationManager.java | 578 +++++++++++---- .../scm/container/balancer/ContainerBalancer.java | 635 ++++++++++++---- .../balancer/ContainerBalancerConfiguration.java | 206 +++++- .../balancer/ContainerBalancerMetrics.java | 13 +- .../ContainerBalancerSelectionCriteria.java | 169 +++++ .../container/balancer/ContainerMoveSelection.java | 55 ++ .../scm/container/balancer/FindTargetGreedy.java | 135 ++++ .../scm/container/balancer/FindTargetStrategy.java | 71 ++ .../algorithms/SCMContainerPlacementRackAware.java | 26 +- .../replication/ReplicationManagerMetrics.java | 161 ++++ .../scm/container/states/ContainerQueryKey.java | 33 +- .../scm/container/states/ContainerStateMap.java | 41 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 7 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++-- .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +- .../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 ++ .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 + .../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 + .../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 + .../apache/hadoop/hdds/scm/node/NodeManager.java | 9 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 27 + ...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 8 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 2 + .../hdds/scm/pipeline/SCMPipelineManager.java | 801 -------------------- ...inerLocationProtocolServerSideTranslatorPB.java | 40 +- .../hdds/scm/server/SCMClientProtocolServer.java | 56 +- .../hdds/scm/server/StorageContainerManager.java | 27 +- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +- .../ScmHAUnfinalizedStateValidationAction.java | 13 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 67 +- .../container/TestCloseContainerEventHandler.java | 6 +- .../hdds/scm/container/TestReplicationManager.java | 474 ++++++++++-- .../scm/container/TestSCMContainerManager.java | 6 +- .../container/balancer/TestContainerBalancer.java | 466 ++++++++++-- .../TestSCMContainerPlacementRandom.java | 68 ++ .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 - .../hdds/scm/node/TestContainerPlacement.java | 4 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 + .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 --------------------- ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 + .../safemode/TestHealthyPipelineSafeModeRule.java | 14 +- .../TestOneReplicaPipelineSafeModeRule.java | 6 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +- .../server/TestSCMUpdateServiceGrpcServer.java | 3 + .../ozone/container/common/TestEndPoint.java | 3 +- .../hdds/scm/cli/ContainerBalancerCommands.java | 19 +- .../scm/cli/ContainerBalancerStartSubcommand.java | 16 +- .../hdds/scm/cli/ContainerOperationClient.java | 26 +- .../hdds/scm/cli/container/ContainerCommands.java | 1 - .../hdds/scm/cli/container/DeleteSubcommand.java | 54 -- .../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 45 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 33 +- .../hadoop/ozone/om/helpers/BucketLayout.java | 62 ++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 1 - .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 5 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +- .../om/helpers/TestOmKeyLocationInfoGroup.java | 19 + hadoop-ozone/dev-support/docker/Dockerfile | 38 +- .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../src/main/compose/ozone-om-ha/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 + .../dist/src/main/compose/ozone/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 + .../src/main/compose/ozonesecure-ha/docker-config | 4 + .../src/main/compose/ozonesecure/docker-config | 4 + hadoop-ozone/dist/src/main/license/jar-report.txt | 47 +- .../dist/src/main/license/update-jar-report.sh | 2 +- .../src/main/smoketest/admincli/container.robot | 4 - .../dist/src/main/smoketest/basic/links.robot | 10 +- .../dist/src/main/smoketest/s3/boto3.robot | 5 +- .../dist/src/main/smoketest/s3/boto_client.py | 10 +- .../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +- .../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +- .../dist/src/main/smoketest/s3/buckethead.robot | 3 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 10 +- .../src/main/smoketest/s3/s3_compatbility_check.sh | 39 + .../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +- .../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 +- .../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 30 +- .../ozone/contract/ITestOzoneContractDistCp.java | 8 + .../rooted/ITestRootedOzoneContractDistCp.java | 8 + .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +- .../TestContainerStateManagerIntegration.java | 40 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 +++++ .../hdds/scm/pipeline/TestPipelineClose.java | 2 + .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +- .../ozone/TestContainerBalancerOperations.java | 11 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +- .../hadoop/ozone/TestStorageContainerManager.java | 10 +- .../client/rpc/TestOzoneAtRestEncryption.java | 130 +++- .../ozone/container/TestContainerReplication.java | 1 - .../commandhandler/TestBlockDeletion.java | 125 +++- .../server/TestSecureContainerServer.java | 6 + .../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +- .../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ----- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 5 +- .../hadoop/ozone/scm/TestCloseContainer.java | 8 + .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 ++- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 19 +- .../scm/node/TestDecommissionAndMaintenance.java | 6 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +- .../src/main/proto/OmClientProtocol.proto | 7 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 31 + .../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +- .../om/request/bucket/OMBucketCreateRequest.java | 102 +-- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +- .../request/s3/security/S3RevokeSecretRequest.java | 2 +- .../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 40 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 258 ++++++- .../hadoop/ozone/recon/api/types/DUResponse.java | 56 ++ .../recon/api/types/NamespaceSummaryResponse.java | 19 +- .../ozone/recon/scm/ReconPipelineManager.java | 4 +- .../recon/scm/ReconPipelineReportHandler.java | 22 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 23 + .../ozone/recon/api/TestNSSummaryEndpoint.java | 168 ++++- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 29 + .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 + .../ozone/genesis/BenchMarkContainerStateMap.java | 6 +- .../ozone/shell/bucket/CreateBucketHandler.java | 20 +- .../ozone/shell/bucket/InfoBucketHandler.java | 52 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +- pom.xml | 6 +- 208 files changed, 7811 insertions(+), 4289 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
