This is an automated email from the ASF dual-hosted git repository.
captainzmc pushed a change to branch HDDS-4454
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from 67c3771 HDDS-5483. Validate block file length during put block (#2463)
add 074d8f4 HDDS-5382. Increase default container report interval to 60
mins (#2363)
add 5da8b22 HDDS-5328. Remove delete container command from admin CLI
(#2456)
add 1c6e0fb HDDS-5279. OFS mkdir -p does not work when Volume is not
pre-created (#2412)
add 210eb3b HDDS-5514. Skip check for UNHEALTHY containers for datanode
finalize. (#2469)
add b447ffc HDDS-5401. Add more metrics to ReplicationManager to help
monitor replication progress. (#2382)
add 622275f HDDS-5513. Race condition upon dn restart at prefinalization.
(#2471)
add 99fa944 HDDS-5453. Add metrics in SCM block deleting service. (#2438)
add d557817 HDDS-5524. Use try-with-resources to properly close resources
in DirectoryServerSource (#2481)
add 8739fd2 HDDS-5512. Intermittent failure in
TestSCMInstallSnapshotWithHA (#2473)
add d0f0f80 HDDS-5501. Support to upload/read keys from encrypted buckets
through S3G (#2467)
add 01c6165 HDDS-5511. ContainerStateMap should handle ecReplication
config map (#2470)
add 7df1925 HDDS-5362. [FSO] Support bucket layouts in OM (#2357)
add 0b5e965 HDDS-5302. CleanUp V1 Code of DeletedBlockLogImpl (#2465)
add e2d1727 HDDS-5472. Old versions of location in OmKeyLocationInfoGroup
causes OOM of OM (#2448)
add fd7e850 HDDS-5460: ReplicationConfig#getDefault is hardcoded with
RatisReplicationConfig (#2430)
add ec031d4 HDDS-5515. SKip update firstUpgradeActionLayoutVersion when
on-disk layout version is up-to-date. (#2472)
add cead31e HDDS-5530. Support links to encrypted buckets. (#2487)
add b3a2a3e HDDS-5441. Disallow same set of DNs to be part of multiple
pipelines. (#2416)
add 30287c7 HDDS-5539. Fix actual value in assertion in TestRDBStore
(#2494)
add 00a5afe HDDS-5354. Allow to restrict available ReplicationConfig
(#2345)
add 4503249 HDDS-5532. Missing integration test cleanup (#2496)
add 181d44a HDDS-5527. Move tests back to root partition (#2486)
add 61396ba HDDS-4668. Intermittent failure in TestOMRatisSnapshots
(#2474)
add 94b8c4b HDDS-5516. Duplicate metrics registered while running
checkScmHA upon scm startup. (#2475)
add e23fb3c HDDS-5473. Bump jetty version to 9.4.43.v20210629 (#2449)
add bc1dfb8 HDDS-5538. Uncaught NPE in GrpcReplicationService. (#2503)
add 063e7fd HDDS-5390. reconPipelineReportHandler should not retry when
pipeline not found (#2371)
add 2020d30 HDDS-5503. On finalize upgrade actions not running. (#2476)
new 6b43b5c Merge remote-tracking branch 'origin/master' into HDDS-4454
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/post-commit.yml | 54 +--
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 2 +-
.../hadoop/hdds/client/ReplicationConfig.java | 10 +-
.../hdds/client/ReplicationConfigValidator.java | 63 +++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 5 +
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 2 +
.../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +-
.../common/src/main/resources/ozone-default.xml | 27 +-
.../hadoop/hdds/client/TestReplicationConfig.java | 24 ++
.../client/TestReplicationConfigValidator.java | 73 ++++
.../common/statemachine/DatanodeStateMachine.java | 8 +-
.../OnDemandContainerReplicationSource.java | 10 +-
.../container/stream/DirectoryServerSource.java | 12 +-
.../upgrade/DataNodeUpgradeFinalizer.java | 4 +-
.../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +-
.../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 14 +
.../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 19 +-
.../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 281 ++++++--------
.../hdds/scm/block/DeletedBlockLogImplV2.java | 432 ---------------------
.../hdds/scm/block/SCMBlockDeletingService.java | 11 +-
.../scm/block/ScmBlockDeletingServiceMetrics.java | 186 +++++++++
.../scm/command/CommandStatusReportHandler.java | 6 +-
.../hdds/scm/container/ContainerStateManager.java | 10 +-
.../hdds/scm/container/ReplicationManager.java | 117 +++---
.../replication/ReplicationManagerMetrics.java | 161 ++++++++
.../scm/container/states/ContainerQueryKey.java | 33 +-
.../scm/container/states/ContainerStateMap.java | 41 +-
.../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +-
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +-
.../apache/hadoop/hdds/scm/node/NodeManager.java | 9 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 27 ++
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 6 +
.../hdds/scm/pipeline/PipelineReportHandler.java | 2 +
.../hdds/scm/server/StorageContainerManager.java | 7 +-
.../scm/server/upgrade/SCMUpgradeFinalizer.java | 3 +-
.../ScmHAUnfinalizedStateValidationAction.java | 13 +-
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +-
.../hdds/scm/container/TestReplicationManager.java | 245 ++++++++++--
.../hdds/scm/cli/container/ContainerCommands.java | 1 -
.../hdds/scm/cli/container/DeleteSubcommand.java | 54 ---
.../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 45 ++-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 33 +-
.../hadoop/ozone/om/helpers/BucketLayout.java | 62 +++
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 ++-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 1 -
.../ozone/om/helpers/OmKeyLocationInfoGroup.java | 5 +-
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +-
.../om/helpers/TestOmKeyLocationInfoGroup.java | 19 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozonesecure-ha/docker-config | 4 +
.../src/main/compose/ozonesecure/docker-config | 4 +
hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +--
.../dist/src/main/license/update-jar-report.sh | 2 +-
.../src/main/smoketest/admincli/container.robot | 4 -
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +-
.../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +-
.../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 55 ++-
.../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 30 +-
.../ozone/contract/ITestOzoneContractDistCp.java | 8 +
.../rooted/ITestRootedOzoneContractDistCp.java | 8 +
.../TestContainerStateManagerIntegration.java | 40 +-
.../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 +++++++++
.../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 130 +++++--
.../commandhandler/TestBlockDeletion.java | 17 +
.../server/TestSecureContainerServer.java | 6 +
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +-
.../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ----------
.../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +++-
.../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +-
.../hadoop/ozone/scm/TestCloseContainer.java | 8 +
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 7 +
.../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +-
.../src/main/proto/OmClientProtocol.proto | 7 +
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +-
.../om/request/bucket/OMBucketCreateRequest.java | 102 ++---
.../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +-
.../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +-
.../ozone/BasicRootedOzoneClientAdapterImpl.java | 40 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +-
.../recon/scm/ReconPipelineReportHandler.java | 22 +-
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 29 ++
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 +
.../ozone/genesis/BenchMarkContainerStateMap.java | 6 +-
.../ozone/shell/bucket/CreateBucketHandler.java | 20 +-
.../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +-
.../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +-
pom.xml | 2 +-
93 files changed, 2111 insertions(+), 1391 deletions(-)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfigValidator.java
create mode 100644
hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfigValidator.java
delete mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/DeletedBlockLogImplV2.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/ScmBlockDeletingServiceMetrics.java
create mode 100644
hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/container/replication/ReplicationManagerMetrics.java
delete mode 100644
hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/DeleteSubcommand.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketLayout.java
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestMultiRaftSetup.java
delete mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMStartupWithLayout.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
