This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a change to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git.
from c050b8f HDDS-5011. Introduce ECReplicationConfig and Java based
ReplicationConfig implementation (#2068)
add e8d001f HDDS-4888. Merge basic CI checks (#1984)
add 8a80c80 HDDS-5090. make Decommission work under SCM HA. (#2148)
add 6da41ca HDDS-5110. TestStorageContainerManagerHttpServer fails in CI
(#2159)
add 014d0a2 HDDS-5075. [SCM HA Security] Remove code of not starting
ozone services when Security is enabled on SCM HA cluster (#2132)
add 8badca9 HDDS-5084. Include HISTORY.md/SECURITY.md/CONTRIBUTING.md in
the release artifacts. (#2140)
add 7afb1f8 HDDS-5091. Add project separation and first stable release to
the HISTORY.md (#2149)
add 724d96c HDDS-5026. Cancel failing PR workflow runs (#2129)
add cda430a HDDS-5087. Ozone RPC client leaks KeyProvider instances.
(#2144)
add 8036536 HDDS-5117. CRLInfo should include CRL Sequence ID (#2161)
add 664837b HDDS-5121. Releasing 1.1.0 from "ozone-1.1" branch. Updating
version on Master.
add 24ecd22 HDDS-5121. Remove duplicate SNAPSHOT from version (#2163)
add 8cdabec HDDS-5116. Secure datanode/OM may exit if it cannot connect
to SCM. (#2162)
add 1761426 HDDS-4986. Read failure because of unhealthy container.
(#2152)
add 057f0a0 HDDS-5061. Improve block commit (#2108)
add c89f08e HDDS-5068. build errors due to missing JAXB-runtime jar
dependency for openjdk 11 se standalone (#2153)
add 9912b69 HDDS-4195. Fix Recon after HDDS-4133. (#2103)
add f2be003 HDDS-5125. Only test ozonesecure with SCM Ratis disabled
(#2168)
add 4e45726 HDDS-5122. SCM Reinitialization can end up leaking Ratis
Segmented RaftLogWorker threads (#2167)
add f2ff682 HDDS-5083. Bump version of common-compress (#2139)
add fb3dee8 HDDS-5047. Refactor Pipeline to use ReplicationConfig instead
of factor/type (#2096)
add 2c183b2 HDDS-5124. Use OzoneConsts.OZONE_TIME_ZONE instead of "GMT"
(#2166)
add bef180e HDDS-5103. Fix Install Snapshot Mechanism in SCMStateMachine.
(#2155)
add 4c313b8 HDDS-5131. Use timeout in github actions (#2176)
add 6af26ca HDDS-4889. Add simple CI check for docs (#2156)
add 7f2dc04 HDDS-5127. Fix getServiceList when SCM HA is enabled (#2173)
add 1e1eefa HDDS-5051. Ensure failover to suggested leader if any for
NotLeaderException. (#2141)
new aeedef8 Merge branch 'master' into the EC branch
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/cancel-ci.yaml | 9 +-
.github/workflows/post-commit.yml | 158 +++++++-------------
HISTORY.md | 4 +-
hadoop-hdds/client/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 +
.../hadoop/hdds/scm/storage/BlockInputStream.java | 7 +-
.../storage/DummyBlockInputStreamWithRetry.java | 7 +-
hadoop-hdds/common/pom.xml | 9 +-
.../java/org/apache/hadoop/hdds/HddsUtils.java | 104 +++++++------
.../hdds/ratis/ServerNotLeaderException.java | 102 +++++++++++++
.../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 -
...erException.java => NonRetriableException.java} | 8 +-
.../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 78 +++++++++-
.../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 5 +-
.../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 67 ++++-----
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 +
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../org/apache/hadoop/ozone/common/Storage.java | 12 ++
.../ozone/container/common/helpers/BlockData.java | 2 +-
.../common/src/main/resources/ozone-default.xml | 15 +-
.../java/org/apache/hadoop/hdds/TestHddsUtils.java | 22 +--
.../hadoop/hdds/scm/pipeline/MockPipeline.java | 12 +-
hadoop-hdds/config/pom.xml | 4 +-
hadoop-hdds/container-service/pom.xml | 12 +-
.../apache/hadoop/ozone/HddsDatanodeService.java | 4 +-
.../common/statemachine/DatanodeStateMachine.java | 2 +-
.../common/states/datanode/InitDatanodeState.java | 4 +-
.../ozone/container/keyvalue/KeyValueHandler.java | 58 ++------
.../container/keyvalue/helpers/BlockUtils.java | 27 +++-
.../container/keyvalue/impl/BlockManagerImpl.java | 16 +-
.../container/common/helpers/TestBlockData.java | 2 +-
.../common/impl/TestContainerDataYaml.java | 13 +-
.../TestKeyValueHandlerWithUnhealthyContainer.java | 18 ++-
.../container/ozoneimpl/TestContainerReader.java | 1 +
hadoop-hdds/docs/pom.xml | 4 +-
hadoop-hdds/framework/pom.xml | 4 +-
.../scm/protocol/ScmBlockLocationProtocol.java | 25 +++-
...lockLocationProtocolClientSideTranslatorPB.java | 53 ++++---
...inerLocationProtocolClientSideTranslatorPB.java | 1 +
.../SCMBlockLocationFailoverProxyProvider.java | 36 ++++-
.../hadoop/hdds/scm/proxy/SCMClientConfig.java | 27 +++-
.../SCMContainerLocationFailoverProxyProvider.java | 63 ++++----
.../SCMSecurityProtocolFailoverProxyProvider.java | 54 +++++--
.../client/DefaultCertificateClient.java | 6 +-
.../hadoop/hdds/security/x509/crl/CRLInfo.java | 24 ++-
.../java/org/apache/hadoop/hdds/utils/HAUtils.java | 25 +---
.../apache/hadoop/hdds/utils/HddsServerUtil.java | 98 ++++++-------
hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +-
hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +-
hadoop-hdds/interface-admin/pom.xml | 4 +-
hadoop-hdds/interface-client/pom.xml | 4 +-
.../interface-client/src/main/proto/hdds.proto | 1 +
hadoop-hdds/interface-server/pom.xml | 4 +-
hadoop-hdds/pom.xml | 4 +-
hadoop-hdds/server-scm/pom.xml | 4 +-
.../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +-
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 45 +++---
.../hdds/scm/container/ContainerManagerImpl.java | 18 +--
.../hdds/scm/container/ContainerManagerV2.java | 9 +-
.../hdds/scm/container/ContainerStateManager.java | 31 ++--
.../hdds/scm/container/SCMContainerManager.java | 6 +-
.../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +-
.../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 14 +-
.../hdds/scm/ha/InterSCMGrpcProtocolService.java | 1 +
.../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 18 +++
.../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 14 +-
.../apache/hadoop/hdds/scm/ha/SCMNodeDetails.java | 4 +
.../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 48 ++----
.../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 2 +-
.../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 28 ++--
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 21 ++-
.../hdds/scm/node/NodeDecommissionManager.java | 26 +++-
.../hadoop/hdds/scm/node/SCMNodeManager.java | 62 +++++---
.../scm/pipeline/BackgroundPipelineCreator.java | 41 ++++--
.../scm/pipeline/BackgroundPipelineCreatorV2.java | 38 +++--
.../hadoop/hdds/scm/pipeline/PipelineFactory.java | 24 ++-
.../hadoop/hdds/scm/pipeline/PipelineManager.java | 40 ++---
.../hdds/scm/pipeline/PipelineManagerV2Impl.java | 74 +++++-----
.../hdds/scm/pipeline/PipelinePlacementPolicy.java | 18 ++-
.../hadoop/hdds/scm/pipeline/PipelineProvider.java | 29 ++--
.../hdds/scm/pipeline/PipelineReportHandler.java | 12 +-
.../hdds/scm/pipeline/PipelineStateManager.java | 34 ++---
.../scm/pipeline/PipelineStateManagerV2Impl.java | 35 +----
.../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 163 +++++----------------
.../hdds/scm/pipeline/RatisPipelineProvider.java | 54 +++----
.../hdds/scm/pipeline/RatisPipelineUtils.java | 14 +-
.../hdds/scm/pipeline/SCMPipelineManager.java | 74 ++++------
.../hdds/scm/pipeline/SCMPipelineMetrics.java | 4 +-
.../hdds/scm/pipeline/SimplePipelineProvider.java | 27 ++--
.../hadoop/hdds/scm/pipeline/StateManager.java | 42 +++---
.../SCMSecurityProtocolServerSideTranslatorPB.java | 17 +--
...lockLocationProtocolServerSideTranslatorPB.java | 28 ++--
...inerLocationProtocolServerSideTranslatorPB.java | 9 +-
.../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +-
.../safemode/OneReplicaPipelineSafeModeRule.java | 20 +--
.../hdds/scm/server/SCMBlockProtocolServer.java | 15 +-
.../hadoop/hdds/scm/server/SCMCertStore.java | 1 +
.../hdds/scm/server/SCMClientProtocolServer.java | 12 +-
.../hadoop/hdds/scm/server/SCMStorageConfig.java | 19 ++-
.../hdds/scm/server/StorageContainerManager.java | 54 ++++---
.../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 31 ++--
.../hadoop/hdds/scm/TestHddsServerUtils.java | 21 ++-
.../scm/TestStorageContainerManagerHttpServer.java | 2 +
.../java/org/apache/hadoop/hdds/scm/TestUtils.java | 21 +--
.../hadoop/hdds/scm/block/TestBlockManager.java | 95 ++++++------
.../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 12 +-
.../container/TestCloseContainerEventHandler.java | 10 +-
.../scm/container/TestContainerManagerImpl.java | 18 ++-
.../scm/container/TestContainerStateManager.java | 15 +-
.../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 7 +-
.../hdds/scm/node/TestNodeDecommissionManager.java | 34 ++++-
.../hadoop/hdds/scm/node/TestSCMNodeManager.java | 27 +++-
.../hdds/scm/pipeline/MockPipelineManager.java | 53 +++----
.../scm/pipeline/MockRatisPipelineProvider.java | 21 +--
.../TestPipelineDatanodesIntersection.java | 13 +-
.../hdds/scm/pipeline/TestPipelineManagerImpl.java | 83 +++++------
.../scm/pipeline/TestPipelinePlacementPolicy.java | 10 +-
.../scm/pipeline/TestPipelineStateManager.java | 101 ++++---------
.../scm/pipeline/TestRatisPipelineProvider.java | 100 +++++++------
.../hdds/scm/pipeline/TestSCMPipelineManager.java | 71 ++++-----
.../scm/pipeline/TestSimplePipelineProvider.java | 28 +++-
.../safemode/TestHealthyPipelineSafeModeRule.java | 26 ++--
.../TestOneReplicaPipelineSafeModeRule.java | 13 +-
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 18 ++-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +-
hadoop-hdds/test-utils/pom.xml | 4 +-
hadoop-hdds/tools/pom.xml | 4 +-
.../hdds/scm/cli/datanode/ListInfoSubcommand.java | 6 +-
.../scm/cli/pipeline/CreatePipelineSubcommand.java | 3 +-
.../scm/cli/pipeline/ListPipelinesSubcommand.java | 4 +-
hadoop-ozone/client/pom.xml | 4 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 63 +++++++-
.../hadoop/ozone/client/TestHddsClientUtils.java | 150 +++++++++++--------
hadoop-ozone/common/pom.xml | 4 +-
.../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 59 +++++++-
.../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 7 +-
hadoop-ozone/csi/pom.xml | 4 +-
hadoop-ozone/datanode/pom.xml | 16 +-
hadoop-ozone/dev-support/checks/_lib.sh | 36 +++++
.../dev-support/checks/{bats.sh => docs.sh} | 20 +--
hadoop-ozone/dev-support/checks/findbugs.sh | 12 +-
.../dist/dev-support/bin/dist-layout-stitching | 3 +
hadoop-ozone/dist/pom.xml | 4 +-
.../dist/src/main/assemblies/ozone-src.xml | 3 +
.../src/main/compose/compatibility/docker-config | 1 +
.../dist/src/main/compose/ozone-csi/docker-config | 1 +
.../dist/src/main/compose/ozone-ha/docker-config | 1 +
.../src/main/compose/ozone-om-ha/docker-config | 1 +
.../src/main/compose/ozone-topology/docker-config | 1 +
.../dist/src/main/compose/ozone/docker-config | 1 +
.../src/main/compose/ozoneblockade/docker-config | 1 +
.../src/main/compose/ozones3-haproxy/docker-config | 1 +
.../src/main/compose/ozonescripts/docker-config | 1 +
.../compose/ozonesecure-ha/docker-compose.yaml | 26 ++++
.../src/main/compose/ozonesecure-ha/docker-config | 2 +-
.../src/main/compose/ozonesecure-mr/docker-config | 1 +
.../main/compose/ozonesecure/docker-compose.yaml | 1 -
.../src/main/compose/ozonesecure/docker-config | 1 +
.../dist/src/main/compose/ozonesecure/test.sh | 51 +++----
.../dist/src/main/compose/restart/docker-config | 1 +
hadoop-ozone/dist/src/main/compose/testlib.sh | 1 -
.../dist/src/main/compose/upgrade/docker-config | 1 +
.../dist/src/main/compose/xcompat/docker-config | 1 +
hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 10 +-
....activation-activation.txt => LICENSE-CDDL.txt} | 1 -
.../src/main/smoketest/admincli/pipeline.robot | 7 +-
.../fault-injection-test/mini-chaos-tests/pom.xml | 4 +-
.../fault-injection-test/network-tests/pom.xml | 2 +-
.../network-tests/src/test/compose/docker-config | 1 +
hadoop-ozone/fault-injection-test/pom.xml | 4 +-
hadoop-ozone/insight/pom.xml | 12 +-
hadoop-ozone/integration-test/pom.xml | 4 +-
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 11 +-
.../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +-
.../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +-
.../metrics/TestSCMContainerManagerMetrics.java | 9 +-
.../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 25 ++--
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 9 +-
.../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 7 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 10 +-
.../TestRatisPipelineCreateAndDestroy.java | 30 ++--
.../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 16 +-
.../safemode/TestSCMSafeModeWithPipelineRules.java | 15 +-
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 10 +-
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +-
.../hadoop/ozone/TestStorageContainerManager.java | 6 +
.../TestContainerStateMachineFailureOnRead.java | 7 +-
.../client/rpc/TestDeleteWithSlowFollower.java | 5 +-
.../client/rpc/TestHybridPipelineOnDatanode.java | 2 +-
.../client/rpc/TestOzoneAtRestEncryption.java | 25 ++++
.../client/rpc/TestOzoneRpcClientAbstract.java | 108 +++++++++++++-
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 5 +
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 13 +-
.../ozone/om/TestOzoneManagerRestInterface.java | 3 +-
.../hadoop/ozone/recon/TestReconAsPassiveScm.java | 13 +-
.../apache/hadoop/ozone/recon/TestReconTasks.java | 5 +-
.../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 147 +++++++++++++++++++
.../TestSCMContainerPlacementPolicyMetrics.java | 7 +-
.../ozone/scm/TestSCMInstallSnapshotWithHA.java | 63 ++------
.../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 9 +-
.../hadoop/ozone/scm/TestXceiverClientGrpc.java | 7 +-
.../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 11 +-
hadoop-ozone/interface-client/pom.xml | 4 +-
hadoop-ozone/interface-storage/pom.xml | 4 +-
hadoop-ozone/ozone-manager/pom.xml | 4 +-
.../org/apache/hadoop/ozone/om/OzoneManager.java | 24 +--
.../hadoop/ozone/om/OzoneManagerStarter.java | 4 -
.../S3MultipartUploadCompleteRequest.java | 2 +-
.../ozone/om/ScmBlockLocationTestingClient.java | 12 +-
.../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +-
.../ozone/om/TestOzoneManagerHttpServer.java | 2 +
.../ozone/om/request/TestOMRequestUtils.java | 54 ++++++-
.../om/request/key/TestOMKeyCommitRequest.java | 78 +++++++++-
.../ozone/om/request/key/TestOMKeyRequest.java | 6 +-
.../om/response/key/TestOMKeyDeleteResponse.java | 4 +-
hadoop-ozone/ozonefs-common/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +-
hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +-
hadoop-ozone/ozonefs-shaded/pom.xml | 4 +-
hadoop-ozone/ozonefs/pom.xml | 4 +-
hadoop-ozone/pom.xml | 4 +-
hadoop-ozone/recon-codegen/pom.xml | 2 +-
hadoop-ozone/recon/pom.xml | 2 +-
.../org/apache/hadoop/ozone/recon/ReconServer.java | 2 -
.../hadoop/ozone/recon/api/NodeEndpoint.java | 6 +-
.../hadoop/ozone/recon/api/PipelineEndpoint.java | 11 +-
.../ozone/recon/scm/ReconPipelineFactory.java | 9 +-
.../recon/scm/ReconPipelineReportHandler.java | 2 +-
.../scm/ReconStorageContainerManagerFacade.java | 15 +-
.../ozone/recon/OMMetadataManagerTestUtils.java | 9 +-
.../hadoop/ozone/recon/api/TestEndpoints.java | 4 +-
.../ozone/recon/scm/TestReconPipelineManager.java | 12 +-
hadoop-ozone/s3gateway/pom.xml | 16 +-
.../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 +-
.../apache/hadoop/ozone/s3/util/RFC1123Util.java | 3 +-
hadoop-ozone/tools/pom.xml | 12 +-
.../StorageContainerManagersCommandHandler.java | 2 +-
.../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 5 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 3 +-
.../hadoop/ozone/freon/DatanodeChunkGenerator.java | 3 +-
.../hadoop/ozone/freon/DatanodeChunkValidator.java | 5 +-
.../ozone/freon/LeaderAppendLogEntryGenerator.java | 6 +-
.../ozone/genesis/BenchMarkContainerStateMap.java | 92 ++++++------
.../ozone/genesis/BenchMarkOzoneManager.java | 5 +-
.../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 7 +-
.../apache/hadoop/ozone/genesis/GenesisUtil.java | 4 +-
pom.xml | 27 ++--
249 files changed, 2961 insertions(+), 2046 deletions(-)
create mode 100644
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/ServerNotLeaderException.java
copy
hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ha/{RetriableWithNoFailoverException.java
=> NonRetriableException.java} (84%)
copy hadoop-ozone/dev-support/checks/{bats.sh => docs.sh} (73%)
copy
hadoop-ozone/dist/src/main/license/bin/licenses/{LICENSE-javax.activation-activation.txt
=> LICENSE-CDDL.txt} (99%)
create mode 100644
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestFailoverWithSCMHA.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
