This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 94814e5 HDDS-5425: Client side pipeline cache key should use host and
port combination (#2510)
add ec031d47 HDDS-5515. SKip update firstUpgradeActionLayoutVersion when
on-disk layout version is up-to-date. (#2472)
add cead31e HDDS-5530. Support links to encrypted buckets. (#2487)
add b3a2a3e HDDS-5441. Disallow same set of DNs to be part of multiple
pipelines. (#2416)
add 30287c7 HDDS-5539. Fix actual value in assertion in TestRDBStore
(#2494)
add 00a5afe HDDS-5354. Allow to restrict available ReplicationConfig
(#2345)
add 4503249 HDDS-5532. Missing integration test cleanup (#2496)
add 181d44a HDDS-5527. Move tests back to root partition (#2486)
add 61396ba HDDS-4668. Intermittent failure in TestOMRatisSnapshots
(#2474)
add 94b8c4b HDDS-5516. Duplicate metrics registered while running
checkScmHA upon scm startup. (#2475)
add e23fb3c HDDS-5473. Bump jetty version to 9.4.43.v20210629 (#2449)
add bc1dfb8 HDDS-5538. Uncaught NPE in GrpcReplicationService. (#2503)
add 063e7fd HDDS-5390. reconPipelineReportHandler should not retry when
pipeline not found (#2371)
add 2020d30 HDDS-5503. On finalize upgrade actions not running. (#2476)
add d51bd4b HDDS-5021. dev-support Dockerfile is badly outdated (#2480)
add f9ec7e9 HDDS-5360. DN failed to process all delete block commands in
one heartbeat interval (#2420)
add 2edb375 HDDS-5546.OM Service ID change causes OM startup failure.
(#2512)
add f922b0d HDDS-5555. remove pipeline manager v1 code (#2511)
add fa7dc30 HDDS-5549. Add 1.1 to supported versions in security policy
(#2519)
add 7eed08f HDDS-5531. For Link Buckets avoid showing metadata. (#2502)
add 1c96a5b HDDS-5558. vUnit invocation unit() may produce NPE (#2513)
add 4625061 HDDS-5449 Recon namespace summary 'du' information should
return replicated size of a key (#2489)
add dc9bb4e HDDS-5283. getStorageSize cast to int can cause issue (#2303)
add a956b49 HDDS-4929. Select target datanodes and containers to move for
Container Balancer (#2441)
add ee929dc HDDS-5000. Run CI checks selectively (#2479)
add 43554e9 HDDS-5608. Fix wrong command in ugrade doc (#2524)
add 44be763 HDDS-5358. Incorrect cache entry invalidation causes
intermittent failure in testGetS3SecretAndRevokeS3Secret (#2518)
add 1045ffc HDDS-5498. Containers are always quasi closed during
finalization. (#2515)
add cef8bf5 HDDS-5508. Add documentation regarding access encrypted
buckets from S3G (#2504)
add 66ee3b4 HDDS-5618. feature/_index.zh.md retranslation (#2535)
add 05c7a98 HDDS-5253. Support container move HA (#2488)
add c9c715c HDDS-5603. Stop ContainerBalancer when SCM is stopped. (#2523)
add 2b70185 HDDS-5533. Update Documentation for Trash (#2505)
add 375f3c4 HDDS-5483. Validate block file length during put block (#2463)
add 8b28e46 HDDS-5458. Extend s3 compatibility script, and fix the
related errors (#2482)
add 8088cf8 HDDS-5601. Publish upgrade documentation on Ozone website.
(#2527)
add bbf9acf HDDS-5054. Merge SCM HA configs to ScmConfigKeys. (#2506)
add d963991 HDDS-5624. Disable test
TestPipelineClose.testPipelineCloseWithLogFailure (#2546)
add 75532de HDDS-5319. Ignore flaky tests in
TestSCMUpdateServiceGrpcServer
add c654ed8 HDDS-5606. Intermittent failure in
TestBlockDeletion#testContainerStatisticsAfterDelete (#2528)
add 800d9a9 HDDS-5175. ozone fs trash service does not handle bucket -rm
-R (#2458)
add bf9fc53 HDDS-5622. Some improvements in SCMCommonPlacementPolicy
(#2543)
add f4ef0e2 HDDS-5621. Support returning nodes of a level in
NetworkTopology (#2541)
add b175d0b HDDS-5437. Move dead DN out of topology (#2435)
new 0e6e00e Merge branch 'master' into ec-branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 130 ++--
SECURITY.md | 1 +
dev-support/ci/lib/_all_libs.sh | 34 +
dev-support/ci/lib/_initialization.sh | 140 ++++
dev-support/ci/lib/_script_init.sh | 42 ++
dev-support/ci/lib/_start_end.sh | 112 +++
dev-support/ci/lib/_traps.sh | 38 +
dev-support/ci/lib/_verbosity.sh | 57 ++
dev-support/ci/selective_ci_checks.bats | 210 ++++++
dev-support/ci/selective_ci_checks.sh | 522 +++++++++++++
.../java/org/apache/hadoop/hdds/HddsUtils.java | 10 +
.../hdds/client/ReplicationConfigValidator.java | 63 ++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 80 ++
.../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +-
.../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 --------
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 6 +-
.../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 +
.../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 +
.../hadoop/hdds/scm/net/NetworkTopology.java | 7 +
.../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 17 +
.../protocol/StorageContainerLocationProtocol.java | 11 +-
.../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +-
.../common/src/main/resources/ozone-default.xml | 123 +++
.../client/TestReplicationConfigValidator.java | 73 ++
.../hdds/scm/net/TestNetworkTopologyImpl.java | 26 +-
.../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 +
.../common/statemachine/DatanodeConfiguration.java | 48 ++
.../common/statemachine/DatanodeStateMachine.java | 3 +-
.../commandhandler/DeleteBlocksCommandHandler.java | 274 +++++--
.../server/ratis/ContainerStateMachine.java | 8 +-
.../transport/server/ratis/XceiverServerRatis.java | 18 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 19 +-
.../OnDemandContainerReplicationSource.java | 10 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 3 +-
.../docs/content/feature/Nonrolling-Upgrade.md | 107 +++
hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +-
hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +-
hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +-
hadoop-hdds/docs/content/feature/_index.zh.md | 14 +-
.../feature/how-to-do-a-nonrolling-upgrade.md | 54 --
hadoop-hdds/docs/content/interface/Ofs.md | 28 +-
hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++
.../container/common/helpers/MoveDataNodePair.java | 71 ++
.../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 +
...inerLocationProtocolClientSideTranslatorPB.java | 30 +-
.../org/apache/hadoop/hdds/server/JsonUtils.java | 2 +
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
.../src/main/proto/ScmAdminProtocol.proto | 4 +-
.../interface-client/src/main/proto/hdds.proto | 5 +
.../src/main/proto/SCMRatisProtocol.proto | 1 +
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 47 +-
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 8 +-
.../hdds/scm/block/SCMBlockDeletingService.java | 5 +-
.../hdds/scm/container/ContainerReplicaCount.java | 1 +
.../hdds/scm/container/ReplicationManager.java | 471 +++++++++---
.../scm/container/balancer/ContainerBalancer.java | 635 ++++++++++++----
.../balancer/ContainerBalancerConfiguration.java | 206 +++++-
.../balancer/ContainerBalancerMetrics.java | 13 +-
.../ContainerBalancerSelectionCriteria.java | 169 +++++
.../container/balancer/ContainerMoveSelection.java | 49 +-
.../scm/container/balancer/FindTargetGreedy.java | 135 ++++
.../scm/container/balancer/FindTargetStrategy.java | 71 ++
.../algorithms/SCMContainerPlacementRackAware.java | 26 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 7 +-
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 131 ++--
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +-
...actionCodec.java => MoveDataNodePairCodec.java} | 36 +-
.../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +-
.../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 +
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 +
.../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 +
.../apache/hadoop/hdds/scm/node/NodeManager.java | 9 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 27 +
...ManagerV2Impl.java => PipelineManagerImpl.java} | 22 +-
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 8 +-
.../hdds/scm/pipeline/PipelineReportHandler.java | 2 +
...inerLocationProtocolServerSideTranslatorPB.java | 40 +-
.../hdds/scm/server/SCMClientProtocolServer.java | 56 +-
.../hdds/scm/server/StorageContainerManager.java | 23 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +-
.../ScmHAUnfinalizedStateValidationAction.java | 13 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 6 +-
.../hadoop/hdds/scm/container/MockNodeManager.java | 67 +-
.../container/TestCloseContainerEventHandler.java | 6 +-
.../hdds/scm/container/TestReplicationManager.java | 229 ++++--
.../scm/container/TestSCMContainerManager.java | 6 +-
.../container/balancer/TestContainerBalancer.java | 466 ++++++++++--
.../TestSCMContainerPlacementRandom.java | 68 ++
.../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 -
.../hdds/scm/node/TestContainerPlacement.java | 4 +-
.../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 +
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +-
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 ---------------------
...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 +
.../safemode/TestHealthyPipelineSafeModeRule.java | 14 +-
.../TestOneReplicaPipelineSafeModeRule.java | 6 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +-
.../server/TestSCMUpdateServiceGrpcServer.java | 3 +
.../ozone/container/common/TestEndPoint.java | 3 +-
.../hdds/scm/cli/ContainerBalancerCommands.java | 19 +-
.../scm/cli/ContainerBalancerStartSubcommand.java | 16 +-
.../hdds/scm/cli/ContainerOperationClient.java | 26 +-
hadoop-ozone/dev-support/docker/Dockerfile | 38 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +-
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../dist/src/main/smoketest/basic/links.robot | 10 +-
.../dist/src/main/smoketest/s3/boto3.robot | 5 +-
.../dist/src/main/smoketest/s3/boto_client.py | 10 +-
.../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +-
.../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +-
.../dist/src/main/smoketest/s3/buckethead.robot | 3 +-
.../dist/src/main/smoketest/s3/commonawslib.robot | 10 +-
.../src/main/smoketest/s3/s3_compatbility_check.sh | 39 +
.../ozone/contract/ITestOzoneContractDistCp.java | 8 +
.../rooted/ITestRootedOzoneContractDistCp.java | 8 +
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 +++++
.../hdds/scm/pipeline/TestPipelineClose.java | 2 +
.../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +-
.../ozone/TestContainerBalancerOperations.java | 11 +-
.../hadoop/ozone/TestStorageContainerManager.java | 10 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 130 +++-
.../ozone/container/TestContainerReplication.java | 1 -
.../commandhandler/TestBlockDeletion.java | 108 ++-
.../server/TestSecureContainerServer.java | 6 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +-
.../ozone/recon/TestReconWithOzoneManagerFSO.java | 5 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 8 +
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 ++-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 12 +-
.../scm/node/TestDecommissionAndMaintenance.java | 6 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 31 +
.../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +-
.../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +-
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +-
.../request/s3/security/S3RevokeSecretRequest.java | 2 +-
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +-
.../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 258 ++++++-
.../hadoop/ozone/recon/api/types/DUResponse.java | 56 ++
.../recon/api/types/NamespaceSummaryResponse.java | 19 +-
.../ozone/recon/scm/ReconPipelineManager.java | 4 +-
.../recon/scm/ReconPipelineReportHandler.java | 22 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 23 +
.../ozone/recon/api/TestNSSummaryEndpoint.java | 168 ++++-
.../ozone/shell/bucket/InfoBucketHandler.java | 52 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +-
pom.xml | 6 +-
153 files changed, 6277 insertions(+), 2274 deletions(-)
create mode 100755 dev-support/ci/lib/_all_libs.sh
create mode 100644 dev-support/ci/lib/_initialization.sh
create mode 100755 dev-support/ci/lib/_script_init.sh
create mode 100644 dev-support/ci/lib/_start_end.sh
create mode 100644 dev-support/ci/lib/_traps.sh
create mode 100644 dev-support/ci/lib/_verbosity.sh
create mode 100644 dev-support/ci/selective_ci_checks.bats
create mode 100755 dev-support/ci/selective_ci_checks.sh
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
delete mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAConfiguration.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
create mode 100644 hadoop-hdds/docs/content/feature/Nonrolling-Upgrade.md
delete mode 100644
hadoop-hdds/docs/content/feature/how-to-do-a-nonrolling-upgrade.md
create mode 100644
hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/container/common/helpers/MoveDataNodePair.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerBalancerSelectionCriteria.java
copy
hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/ContainerMetadata.java
=>
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/ContainerMoveSelection.java
(54%)
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetGreedy.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/balancer/FindTargetStrategy.java
copy
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/metadata/{DeletedBlocksTransactionCodec.java
=> MoveDataNodePairCodec.java} (55%)
rename
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/{PipelineManagerV2Impl.java
=> PipelineManagerImpl.java} (96%)
delete mode 100644
hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineManager.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestMultiRaftSetup.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
