szetszwo commented on code in PR #12061: URL: https://github.com/apache/ozone/pull/12061#discussion_r4654917412
########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ChainedPipelineProvider.java: ########## @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.hdds.client.ChainedReplicationConfig; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.ha.SCMContext; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Implements Api for creating CHAINED (V3) pipelines. + * Only factor THREE is supported. + * All the datanodes in a CHAINED pipeline must have the + * {@link DatanodeDetails.Port.Name#RATIS_DATASTREAM} port. + */ +public class ChainedPipelineProvider extends NCopyPipelineProvider<ChainedReplicationConfig> { + + public ChainedPipelineProvider(NodeManager nodeManager, + PipelineStateManager stateManager, + ConfigurationSource conf, + EventPublisher eventPublisher, + SCMContext scmContext) { + super(nodeManager, stateManager, conf, eventPublisher, scmContext); + } + + @Override + public synchronized Pipeline create(ChainedReplicationConfig replicationConfig, + List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes) + throws IOException { + final ReplicationFactor factor = replicationConfig.getReplicationFactor(); + if (factor != ReplicationFactor.THREE) { + throw new IllegalArgumentException("Unsupported factor " + factor + " for " + replicationConfig); + } Review Comment: We should allow ReplicationFactor.ONE also. Then V3 pipelines can completely replace V1/V2 pipelines in the future. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/NCopyPipelineProvider.java: ########## @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import static org.apache.hadoop.hdds.protocol.DatanodeDetails.Port.Name.RATIS_DATASTREAM; + +import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hdds.client.ReplicatedReplicationConfig; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.StorageUnit; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.events.SCMEvents; +import org.apache.hadoop.hdds.scm.exceptions.SCMException; +import org.apache.hadoop.hdds.scm.ha.SCMContext; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicy; +import org.apache.hadoop.hdds.scm.pipeline.leader.choose.algorithms.LeaderChoosePolicyFactory; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.protocol.commands.ClosePipelineCommand; +import org.apache.hadoop.ozone.protocol.commands.CommandForDatanode; +import org.apache.hadoop.ozone.protocol.commands.CreatePipelineCommand; +import org.apache.ratis.protocol.exceptions.NotLeaderException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Base class for providers of pipelines with N copies, such as RATIS and CHAINED pipelines. + * The datanodes are chosen by {@link PipelinePlacementPolicy}, + * a leader is suggested by {@link LeaderChoosePolicy} + * and the pipelines are created and closed on the datanodes using + * {@link CreatePipelineCommand} and {@link ClosePipelineCommand}. + */ +public abstract class NCopyPipelineProvider<C extends ReplicatedReplicationConfig> extends PipelineProvider<C> { + + private static final Logger LOG = LoggerFactory.getLogger(NCopyPipelineProvider.class); + + private final EventPublisher eventPublisher; + private final PlacementPolicy placementPolicy; + private final LeaderChoosePolicy leaderChoosePolicy; + private final SCMContext scmContext; + private final long containerSizeBytes; + private final long minRatisVolumeSizeBytes; + + protected NCopyPipelineProvider(NodeManager nodeManager, + PipelineStateManager stateManager, + ConfigurationSource conf, + EventPublisher eventPublisher, + SCMContext scmContext) { + super(nodeManager, stateManager); + this.eventPublisher = eventPublisher; + this.scmContext = scmContext; + this.placementPolicy = PipelinePlacementPolicyFactory + .getPolicy(nodeManager, stateManager, conf); + this.containerSizeBytes = (long) conf.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); + this.minRatisVolumeSizeBytes = (long) conf.getStorageSize( + ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN, + ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN_DEFAULT, + StorageUnit.BYTES); + try { + leaderChoosePolicy = LeaderChoosePolicyFactory + .getPolicy(conf, nodeManager, stateManager); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @VisibleForTesting + public LeaderChoosePolicy getLeaderChoosePolicy() { + return leaderChoosePolicy; + } + + long getContainerSizeBytes() { + return containerSizeBytes; + } + + long getMinRatisVolumeSizeBytes() { + return minRatisVolumeSizeBytes; + } + + @Override + public synchronized Pipeline create(C replicationConfig) throws IOException { + return create(replicationConfig, Collections.emptyList(), Collections.emptyList()); + } + + @Override + public Pipeline create(C replicationConfig, List<DatanodeDetails> nodes) { + return newPipelineBuilder(replicationConfig, nodes) + .setId(PipelineID.randomId()) + .build(); + } + + @Override + public Pipeline createForRead(C replicationConfig, Set<ContainerReplica> replicas) { + // Use insecureRandomId for throwaway read pipeline IDs to avoid + // contention on the shared SecureRandom instance. + return newPipelineBuilder(replicationConfig, ContainerReplica.toDatanodeDetailsList(replicas)) + .setId(PipelineID.insecureRandomId()) + .build(); + } + + /** + * Build a new pipeline with the given datanodes and a suggested leader. + * Then, send {@link CreatePipelineCommand} to each datanode. + */ + Pipeline createPipeline(C replicationConfig, List<DatanodeDetails> dns) throws NotLeaderException { + DatanodeDetails suggestedLeader = leaderChoosePolicy.chooseLeader(dns); + + Pipeline pipeline = newPipelineBuilder(replicationConfig, dns) + .setId(PipelineID.randomId()) + .setSuggestedLeaderId(suggestedLeader != null ? suggestedLeader.getID() : null) + .build(); + + // Send command to datanodes to create pipeline + final ReplicationFactor factor = replicationConfig.getReplicationFactor(); + final CreatePipelineCommand createCommand = new CreatePipelineCommand( + pipeline.getId(), pipeline.getType(), factor, dns, suggestedLeader); + createCommand.setTerm(scmContext.getTermOfLeader()); + + dns.forEach(node -> { + LOG.info("Sending CreatePipelineCommand for pipeline:{} to datanode:{}", + pipeline.getId(), node); + eventPublisher.fireEvent(SCMEvents.DATANODE_COMMAND, + new CommandForDatanode<>(node, createCommand)); + }); + + return pipeline; + } + + /** + * Choose datanodes for a factor THREE pipeline, + * excluding the datanodes which have reached their pipeline limit. + * + * @param excludeNonStreamNodes Should the datanodes without the {@link DatanodeDetails.Port.Name#RATIS_DATASTREAM} + * port be excluded? + * @param fallback When excludeNonStreamNodes is true, should it fall back to choose from all the datanodes + * if there are not enough datanodes with the stream port? + */ + List<DatanodeDetails> chooseThreeFactorDatanodes( + List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes, int requiredNode, + boolean excludeNonStreamNodes, boolean fallback) + throws IOException { + final NodeManager nodeManager = getNodeManager(); + final PipelineStateManager stateManager = getPipelineStateManager(); + final List<DatanodeDetails> healthyNodes = nodeManager.getNodes(NodeStatus.inServiceHealthy()); + excludedNodes = excludedNodes.isEmpty() ? null : excludedNodes; + List<DatanodeDetails> additionalExcludedNodes = null; + for (DatanodeDetails d : healthyNodes) { + final int count = PipelinePlacementPolicy.currentRatisThreePipelineCount(nodeManager, stateManager, d); + if (count >= nodeManager.pipelineLimit(d)) { Review Comment: - currentRatisThreePipelineCount filtered non-RATIS pipeline in isNonClosedRatisThreePipeline. So CHAINED pipelines are not counted. - We need separated pipelineLimit for RATIS and CHAINED. ########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ChainedPipelineProvider.java: ########## @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.scm.pipeline; + +import java.io.IOException; +import java.util.List; +import org.apache.hadoop.hdds.client.ChainedReplicationConfig; +import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; +import org.apache.hadoop.hdds.scm.ha.SCMContext; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.server.events.EventPublisher; + +/** + * Implements Api for creating CHAINED (V3) pipelines. + * Only factor THREE is supported. + * All the datanodes in a CHAINED pipeline must have the + * {@link DatanodeDetails.Port.Name#RATIS_DATASTREAM} port. + */ +public class ChainedPipelineProvider extends NCopyPipelineProvider<ChainedReplicationConfig> { + + public ChainedPipelineProvider(NodeManager nodeManager, + PipelineStateManager stateManager, + ConfigurationSource conf, + EventPublisher eventPublisher, + SCMContext scmContext) { + super(nodeManager, stateManager, conf, eventPublisher, scmContext); + } + + @Override + public synchronized Pipeline create(ChainedReplicationConfig replicationConfig, + List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes) + throws IOException { Review Comment: Check HDDSVersion.STREAM_PUTBLOCK_WITHOUT_RAFT and disallow CHAINED pipeline before upgrade finalized. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
