This is an automated email from the ASF dual-hosted git repository. sodonnell pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0e6e00e97ec1f8d07092df32a529061acd7df1b2 Merge: 94814e5 b175d0b Author: S O'Donnell <[email protected]> AuthorDate: Wed Aug 18 15:44:32 2021 +0100 Merge branch 'master' into ec-branch .github/workflows/post-commit.yml | 130 ++-- SECURITY.md | 1 + dev-support/ci/lib/_all_libs.sh | 34 + dev-support/ci/lib/_initialization.sh | 140 ++++ dev-support/ci/lib/_script_init.sh | 42 ++ dev-support/ci/lib/_start_end.sh | 112 +++ dev-support/ci/lib/_traps.sh | 38 + dev-support/ci/lib/_verbosity.sh | 57 ++ dev-support/ci/selective_ci_checks.bats | 210 ++++++ dev-support/ci/selective_ci_checks.sh | 522 +++++++++++++ .../java/org/apache/hadoop/hdds/HddsUtils.java | 10 + .../hdds/client/ReplicationConfigValidator.java | 63 ++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++ .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 -------- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +- .../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 + .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 + .../hadoop/hdds/scm/net/NetworkTopology.java | 7 + .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 + .../protocol/StorageContainerLocationProtocol.java | 11 +- .../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +- .../common/src/main/resources/ozone-default.xml | 123 +++ .../client/TestReplicationConfigValidator.java | 73 ++ .../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +- .../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 + .../common/statemachine/DatanodeConfiguration.java | 48 ++ .../common/statemachine/DatanodeStateMachine.java | 3 +- .../commandhandler/DeleteBlocksCommandHandler.java | 274 +++++-- .../server/ratis/ContainerStateMachine.java | 8 +- .../transport/server/ratis/XceiverServerRatis.java | 18 +- .../ozone/container/keyvalue/KeyValueHandler.java | 19 +- .../OnDemandContainerReplicationSource.java | 10 +- .../upgrade/DataNodeUpgradeFinalizer.java | 3 +- .../docs/content/feature/Nonrolling-Upgrade.md | 107 +++ hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +- hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/_index.zh.md | 14 +- .../feature/how-to-do-a-nonrolling-upgrade.md | 54 -- hadoop-hdds/docs/content/interface/Ofs.md | 28 +- hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++ .../container/common/helpers/MoveDataNodePair.java | 71 ++ .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 + ...inerLocationProtocolClientSideTranslatorPB.java | 30 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 2 + .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +- .../src/main/proto/ScmAdminProtocol.proto | 4 +- .../interface-client/src/main/proto/hdds.proto | 5 + .../src/main/proto/SCMRatisProtocol.proto | 1 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 47 +- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 8 +- .../hdds/scm/block/SCMBlockDeletingService.java | 5 +- .../hdds/scm/container/ContainerReplicaCount.java | 1 + .../hdds/scm/container/ReplicationManager.java | 471 +++++++++--- .../scm/container/balancer/ContainerBalancer.java | 635 ++++++++++++---- .../balancer/ContainerBalancerConfiguration.java | 206 +++++- .../balancer/ContainerBalancerMetrics.java | 13 +- .../ContainerBalancerSelectionCriteria.java | 169 +++++ .../container/balancer/ContainerMoveSelection.java | 55 ++ .../scm/container/balancer/FindTargetGreedy.java | 135 ++++ .../scm/container/balancer/FindTargetStrategy.java | 71 ++ .../algorithms/SCMContainerPlacementRackAware.java | 26 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 7 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++-- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +- .../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 ++ .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 + .../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 + .../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 + .../apache/hadoop/hdds/scm/node/NodeManager.java | 9 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 27 + ...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 8 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 2 + ...inerLocationProtocolServerSideTranslatorPB.java | 40 +- .../hdds/scm/server/SCMClientProtocolServer.java | 56 +- .../hdds/scm/server/StorageContainerManager.java | 23 +- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +- .../ScmHAUnfinalizedStateValidationAction.java | 13 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 67 +- .../container/TestCloseContainerEventHandler.java | 6 +- .../hdds/scm/container/TestReplicationManager.java | 229 ++++-- .../scm/container/TestSCMContainerManager.java | 6 +- .../container/balancer/TestContainerBalancer.java | 466 ++++++++++-- .../TestSCMContainerPlacementRandom.java | 68 ++ .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 - .../hdds/scm/node/TestContainerPlacement.java | 4 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 + .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 --------------------- ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 + .../safemode/TestHealthyPipelineSafeModeRule.java | 14 +- .../TestOneReplicaPipelineSafeModeRule.java | 6 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +- .../server/TestSCMUpdateServiceGrpcServer.java | 3 + .../ozone/container/common/TestEndPoint.java | 3 +- .../hdds/scm/cli/ContainerBalancerCommands.java | 19 +- .../scm/cli/ContainerBalancerStartSubcommand.java | 16 +- .../hdds/scm/cli/ContainerOperationClient.java | 26 +- hadoop-ozone/dev-support/docker/Dockerfile | 38 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +- .../dist/src/main/license/update-jar-report.sh | 2 +- .../dist/src/main/smoketest/basic/links.robot | 10 +- .../dist/src/main/smoketest/s3/boto3.robot | 5 +- .../dist/src/main/smoketest/s3/boto_client.py | 10 +- .../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +- .../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +- .../dist/src/main/smoketest/s3/buckethead.robot | 3 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 10 +- .../src/main/smoketest/s3/s3_compatbility_check.sh | 39 + .../ozone/contract/ITestOzoneContractDistCp.java | 8 + .../rooted/ITestRootedOzoneContractDistCp.java | 8 + .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 +++++ .../hdds/scm/pipeline/TestPipelineClose.java | 2 + .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +- .../ozone/TestContainerBalancerOperations.java | 11 +- .../hadoop/ozone/TestStorageContainerManager.java | 10 +- .../client/rpc/TestOzoneAtRestEncryption.java | 130 +++- .../ozone/container/TestContainerReplication.java | 1 - .../commandhandler/TestBlockDeletion.java | 108 ++- .../server/TestSecureContainerServer.java | 6 + .../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 5 +- .../hadoop/ozone/scm/TestCloseContainer.java | 8 + .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 ++- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 12 +- .../scm/node/TestDecommissionAndMaintenance.java | 6 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 31 + .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +- .../request/s3/security/S3RevokeSecretRequest.java | 2 +- .../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +- .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 258 ++++++- .../hadoop/ozone/recon/api/types/DUResponse.java | 56 ++ .../recon/api/types/NamespaceSummaryResponse.java | 19 +- .../ozone/recon/scm/ReconPipelineManager.java | 4 +- .../recon/scm/ReconPipelineReportHandler.java | 22 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 23 + .../ozone/recon/api/TestNSSummaryEndpoint.java | 168 ++++- .../ozone/shell/bucket/InfoBucketHandler.java | 52 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +- pom.xml | 6 +- 153 files changed, 6344 insertions(+), 2230 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
