This is an automated email from the ASF dual-hosted git repository. sodonnell pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit aeedef8c7b7ae6ac73ea25b0d1e556abe3813017 Merge: c050b8f 1e1eefa Author: S O'Donnell <[email protected]> AuthorDate: Fri Apr 30 14:38:03 2021 +0100 Merge branch 'master' into the EC branch .github/workflows/cancel-ci.yaml | 9 +- .github/workflows/post-commit.yml | 158 +++++++------------- HISTORY.md | 4 +- hadoop-hdds/client/pom.xml | 4 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 + .../hadoop/hdds/scm/storage/BlockInputStream.java | 7 +- .../storage/DummyBlockInputStreamWithRetry.java | 7 +- hadoop-hdds/common/pom.xml | 9 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 104 +++++++------ .../hdds/ratis/ServerNotLeaderException.java | 102 +++++++++++++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 - .../hadoop/hdds/scm/ha/NonRetriableException.java | 30 ++++ .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 78 +++++++++- .../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 5 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 67 ++++----- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../org/apache/hadoop/ozone/common/Storage.java | 12 ++ .../ozone/container/common/helpers/BlockData.java | 2 +- .../common/src/main/resources/ozone-default.xml | 15 +- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 22 +-- .../hadoop/hdds/scm/pipeline/MockPipeline.java | 12 +- hadoop-hdds/config/pom.xml | 4 +- hadoop-hdds/container-service/pom.xml | 12 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 4 +- .../common/statemachine/DatanodeStateMachine.java | 2 +- .../common/states/datanode/InitDatanodeState.java | 4 +- .../ozone/container/keyvalue/KeyValueHandler.java | 58 ++------ .../container/keyvalue/helpers/BlockUtils.java | 27 +++- .../container/keyvalue/impl/BlockManagerImpl.java | 16 +- .../container/common/helpers/TestBlockData.java | 2 +- .../common/impl/TestContainerDataYaml.java | 13 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 18 ++- .../container/ozoneimpl/TestContainerReader.java | 1 + hadoop-hdds/docs/pom.xml | 4 +- hadoop-hdds/framework/pom.xml | 4 +- .../scm/protocol/ScmBlockLocationProtocol.java | 25 +++- ...lockLocationProtocolClientSideTranslatorPB.java | 53 ++++--- ...inerLocationProtocolClientSideTranslatorPB.java | 1 + .../SCMBlockLocationFailoverProxyProvider.java | 36 ++++- .../hadoop/hdds/scm/proxy/SCMClientConfig.java | 27 +++- .../SCMContainerLocationFailoverProxyProvider.java | 63 ++++---- .../SCMSecurityProtocolFailoverProxyProvider.java | 54 +++++-- .../client/DefaultCertificateClient.java | 6 +- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 24 ++- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 25 +--- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 98 ++++++------- hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +- hadoop-hdds/interface-admin/pom.xml | 4 +- hadoop-hdds/interface-client/pom.xml | 4 +- .../interface-client/src/main/proto/hdds.proto | 1 + hadoop-hdds/interface-server/pom.xml | 4 +- hadoop-hdds/pom.xml | 4 +- hadoop-hdds/server-scm/pom.xml | 4 +- .../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 45 +++--- .../hdds/scm/container/ContainerManagerImpl.java | 18 +-- .../hdds/scm/container/ContainerManagerV2.java | 9 +- .../hdds/scm/container/ContainerStateManager.java | 31 ++-- .../hdds/scm/container/SCMContainerManager.java | 6 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 14 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 1 + .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 18 +++ .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 14 +- .../apache/hadoop/hdds/scm/ha/SCMNodeDetails.java | 4 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 48 ++---- .../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 2 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 28 ++-- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 21 ++- .../hdds/scm/node/NodeDecommissionManager.java | 26 +++- .../hadoop/hdds/scm/node/SCMNodeManager.java | 62 +++++--- .../scm/pipeline/BackgroundPipelineCreator.java | 41 ++++-- .../scm/pipeline/BackgroundPipelineCreatorV2.java | 38 +++-- .../hadoop/hdds/scm/pipeline/PipelineFactory.java | 24 ++- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 40 ++--- .../hdds/scm/pipeline/PipelineManagerV2Impl.java | 74 +++++----- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 18 ++- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 29 ++-- .../hdds/scm/pipeline/PipelineReportHandler.java | 12 +- .../hdds/scm/pipeline/PipelineStateManager.java | 34 ++--- .../scm/pipeline/PipelineStateManagerV2Impl.java | 35 +---- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 163 +++++---------------- .../hdds/scm/pipeline/RatisPipelineProvider.java | 54 +++---- .../hdds/scm/pipeline/RatisPipelineUtils.java | 14 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 74 ++++------ .../hdds/scm/pipeline/SCMPipelineMetrics.java | 4 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 27 ++-- .../hadoop/hdds/scm/pipeline/StateManager.java | 42 +++--- .../SCMSecurityProtocolServerSideTranslatorPB.java | 17 +-- ...lockLocationProtocolServerSideTranslatorPB.java | 28 ++-- ...inerLocationProtocolServerSideTranslatorPB.java | 9 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +- .../safemode/OneReplicaPipelineSafeModeRule.java | 20 +-- .../hdds/scm/server/SCMBlockProtocolServer.java | 15 +- .../hadoop/hdds/scm/server/SCMCertStore.java | 1 + .../hdds/scm/server/SCMClientProtocolServer.java | 12 +- .../hadoop/hdds/scm/server/SCMStorageConfig.java | 19 ++- .../hdds/scm/server/StorageContainerManager.java | 54 ++++--- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 31 ++-- .../hadoop/hdds/scm/TestHddsServerUtils.java | 21 ++- .../scm/TestStorageContainerManagerHttpServer.java | 2 + .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 21 +-- .../hadoop/hdds/scm/block/TestBlockManager.java | 95 ++++++------ .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 12 +- .../container/TestCloseContainerEventHandler.java | 10 +- .../scm/container/TestContainerManagerImpl.java | 18 ++- .../scm/container/TestContainerStateManager.java | 15 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 7 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 34 ++++- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 27 +++- .../hdds/scm/pipeline/MockPipelineManager.java | 53 +++---- .../scm/pipeline/MockRatisPipelineProvider.java | 21 +-- .../TestPipelineDatanodesIntersection.java | 13 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 83 +++++------ .../scm/pipeline/TestPipelinePlacementPolicy.java | 10 +- .../scm/pipeline/TestPipelineStateManager.java | 101 ++++--------- .../scm/pipeline/TestRatisPipelineProvider.java | 100 +++++++------ .../hdds/scm/pipeline/TestSCMPipelineManager.java | 71 ++++----- .../scm/pipeline/TestSimplePipelineProvider.java | 28 +++- .../safemode/TestHealthyPipelineSafeModeRule.java | 26 ++-- .../TestOneReplicaPipelineSafeModeRule.java | 13 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 18 ++- .../hadoop/hdds/scm/server/TestSCMCertStore.java | 6 +- hadoop-hdds/test-utils/pom.xml | 4 +- hadoop-hdds/tools/pom.xml | 4 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 6 +- .../scm/cli/pipeline/CreatePipelineSubcommand.java | 3 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 4 +- hadoop-ozone/client/pom.xml | 4 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 63 +++++++- .../hadoop/ozone/client/TestHddsClientUtils.java | 150 +++++++++++-------- hadoop-ozone/common/pom.xml | 4 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 59 +++++++- .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 7 +- hadoop-ozone/csi/pom.xml | 4 +- hadoop-ozone/datanode/pom.xml | 16 +- hadoop-ozone/dev-support/checks/_lib.sh | 36 +++++ .../dev-support/checks/docs.sh | 44 +++--- hadoop-ozone/dev-support/checks/findbugs.sh | 12 +- .../dist/dev-support/bin/dist-layout-stitching | 3 + hadoop-ozone/dist/pom.xml | 4 +- .../dist/src/main/assemblies/ozone-src.xml | 3 + .../src/main/compose/compatibility/docker-config | 1 + .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../src/main/compose/ozone-om-ha/docker-config | 1 + .../src/main/compose/ozone-topology/docker-config | 1 + .../dist/src/main/compose/ozone/docker-config | 1 + .../src/main/compose/ozoneblockade/docker-config | 1 + .../src/main/compose/ozones3-haproxy/docker-config | 1 + .../src/main/compose/ozonescripts/docker-config | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 26 ++++ .../src/main/compose/ozonesecure-ha/docker-config | 2 +- .../src/main/compose/ozonesecure-mr/docker-config | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 1 - .../src/main/compose/ozonesecure/docker-config | 1 + .../dist/src/main/compose/ozonesecure/test.sh | 51 +++---- .../dist/src/main/compose/restart/docker-config | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 1 - .../dist/src/main/compose/upgrade/docker-config | 1 + .../dist/src/main/compose/xcompat/docker-config | 1 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 10 +- .../src/main/license/bin/licenses/LICENSE-CDDL.txt | 133 +++++++++++++++++ .../src/main/smoketest/admincli/pipeline.robot | 7 +- .../fault-injection-test/mini-chaos-tests/pom.xml | 4 +- .../fault-injection-test/network-tests/pom.xml | 2 +- .../network-tests/src/test/compose/docker-config | 1 + hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 12 +- hadoop-ozone/integration-test/pom.xml | 4 +- .../hadoop/hdds/scm/TestRatisPipelineLeader.java | 11 +- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +- .../metrics/TestSCMContainerManagerMetrics.java | 9 +- .../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 25 ++-- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 9 +- .../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 7 +- .../hdds/scm/pipeline/TestPipelineClose.java | 10 +- .../TestRatisPipelineCreateAndDestroy.java | 30 ++-- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 16 +- .../safemode/TestSCMSafeModeWithPipelineRules.java | 15 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 10 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +- .../hadoop/ozone/TestStorageContainerManager.java | 6 + .../TestContainerStateMachineFailureOnRead.java | 7 +- .../client/rpc/TestDeleteWithSlowFollower.java | 5 +- .../client/rpc/TestHybridPipelineOnDatanode.java | 2 +- .../client/rpc/TestOzoneAtRestEncryption.java | 25 ++++ .../client/rpc/TestOzoneRpcClientAbstract.java | 108 +++++++++++++- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 5 + .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 13 +- .../ozone/om/TestOzoneManagerRestInterface.java | 3 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 13 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 5 +- .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 147 +++++++++++++++++++ .../TestSCMContainerPlacementPolicyMetrics.java | 7 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 63 ++------ .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 9 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 7 +- .../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 11 +- hadoop-ozone/interface-client/pom.xml | 4 +- hadoop-ozone/interface-storage/pom.xml | 4 +- hadoop-ozone/ozone-manager/pom.xml | 4 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 24 +-- .../hadoop/ozone/om/OzoneManagerStarter.java | 4 - .../S3MultipartUploadCompleteRequest.java | 2 +- .../ozone/om/ScmBlockLocationTestingClient.java | 12 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +- .../ozone/om/TestOzoneManagerHttpServer.java | 2 + .../ozone/om/request/TestOMRequestUtils.java | 54 ++++++- .../om/request/key/TestOMKeyCommitRequest.java | 78 +++++++++- .../ozone/om/request/key/TestOMKeyRequest.java | 6 +- .../om/response/key/TestOMKeyDeleteResponse.java | 4 +- hadoop-ozone/ozonefs-common/pom.xml | 4 +- hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +- hadoop-ozone/ozonefs-shaded/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 4 +- hadoop-ozone/pom.xml | 4 +- hadoop-ozone/recon-codegen/pom.xml | 2 +- hadoop-ozone/recon/pom.xml | 2 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 2 - .../hadoop/ozone/recon/api/NodeEndpoint.java | 6 +- .../hadoop/ozone/recon/api/PipelineEndpoint.java | 11 +- .../ozone/recon/scm/ReconPipelineFactory.java | 9 +- .../recon/scm/ReconPipelineReportHandler.java | 2 +- .../scm/ReconStorageContainerManagerFacade.java | 15 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 9 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 4 +- .../ozone/recon/scm/TestReconPipelineManager.java | 12 +- hadoop-ozone/s3gateway/pom.xml | 16 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 +- .../apache/hadoop/ozone/s3/util/RFC1123Util.java | 3 +- hadoop-ozone/tools/pom.xml | 12 +- .../StorageContainerManagersCommandHandler.java | 2 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 5 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 3 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 3 +- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 5 +- .../ozone/freon/LeaderAppendLogEntryGenerator.java | 6 +- .../ozone/genesis/BenchMarkContainerStateMap.java | 92 ++++++------ .../ozone/genesis/BenchMarkOzoneManager.java | 5 +- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 7 +- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 4 +- pom.xml | 27 ++-- 249 files changed, 3133 insertions(+), 2052 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index f971bc0,d0cf5fd..90a6d94 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@@ -189,22 -175,8 +178,23 @@@ public final class Pipeline return getNodeSet().equals(pipeline.getNodeSet()); } + /** + * Return the replica index of the specific datanode in the datanode set. + * <p> + * For non-EC case all the replication should be exactly the same, + * therefore the replication index can always be zero. In case of EC + * different Datanodes can have different data for one specific block + * (parity and/or data parts) therefore the replicaIndex should be + * different for each of the pipeline members. + * + * @param dn datanode details + */ + public int getReplicaIndex(DatanodeDetails dn) { + return replicaIndexes.getOrDefault(dn, 0); + } + + /** * Returns the leader if found else defaults to closest node. * * @return {@link DatanodeDetails} @@@ -284,15 -256,15 +274,19 @@@ return nodeStatus.isEmpty(); } + public ReplicationConfig getReplicationConfig() { + return replicationConfig; + } + public HddsProtos.Pipeline getProtobufMessage(int clientVersion) throws UnknownPipelineStateException { + List<HddsProtos.DatanodeDetailsProto> members = new ArrayList<>(); + List<Integer> memberReplicaIndexes = new ArrayList<>(); + for (DatanodeDetails dn : nodeStatus.keySet()) { members.add(dn.toProto(clientVersion)); + memberReplicaIndexes.add(replicaIndexes.getOrDefault(dn, 0)); } HddsProtos.Pipeline.Builder builder = HddsProtos.Pipeline.newBuilder() @@@ -510,16 -467,9 +495,15 @@@ return this; } + + public Builder setReplicaIndexes(Map<DatanodeDetails, Integer> indexes) { + this.replicaIndexes = indexes; + return this; + } + public Pipeline build() { Preconditions.checkNotNull(id); - Preconditions.checkNotNull(type); - Preconditions.checkNotNull(factor); + Preconditions.checkNotNull(replicationConfig); Preconditions.checkNotNull(state); Preconditions.checkNotNull(nodeStatus); Pipeline pipeline = --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
