Pengzna commented on code in PR #12355: URL: https://github.com/apache/iotdb/pull/12355#discussion_r1617102941
########## iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/PipeConsensusServerImpl.java: ########## @@ -0,0 +1,545 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.consensus.pipe; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.consensus.index.ComparableConsensusRequest; +import org.apache.iotdb.commons.consensus.index.ProgressIndex; +import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex; +import org.apache.iotdb.commons.pipe.task.meta.PipeStatus; +import org.apache.iotdb.consensus.IStateMachine; +import org.apache.iotdb.consensus.common.DataSet; +import org.apache.iotdb.consensus.common.Peer; +import org.apache.iotdb.consensus.common.request.IConsensusRequest; +import org.apache.iotdb.consensus.config.PipeConsensusConfig; +import org.apache.iotdb.consensus.exception.ConsensusGroupModifyPeerException; +import org.apache.iotdb.consensus.pipe.client.SyncPipeConsensusServiceClient; +import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeManager; +import org.apache.iotdb.consensus.pipe.consensuspipe.ConsensusPipeName; +import org.apache.iotdb.consensus.pipe.consensuspipe.ProgressIndexManager; +import org.apache.iotdb.consensus.pipe.thrift.TCheckConsensusPipeCompleteddReq; +import org.apache.iotdb.consensus.pipe.thrift.TCheckConsensusPipeCompleteddResp; +import org.apache.iotdb.consensus.pipe.thrift.TNotifyPeerToCreateConsensusPipeReq; +import org.apache.iotdb.consensus.pipe.thrift.TNotifyPeerToCreateConsensusPipeResp; +import org.apache.iotdb.consensus.pipe.thrift.TNotifyPeerToDropConsensusPipeReq; +import org.apache.iotdb.consensus.pipe.thrift.TNotifyPeerToDropConsensusPipeResp; +import org.apache.iotdb.consensus.pipe.thrift.TSetActiveReq; +import org.apache.iotdb.consensus.pipe.thrift.TSetActiveResp; +import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.rpc.RpcUtils; + +import com.google.common.collect.ImmutableMap; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +/** PipeConsensusServerImpl is a consensus server implementation for pipe consensus. */ +public class PipeConsensusServerImpl { + private static final Logger LOGGER = LoggerFactory.getLogger(PipeConsensusServerImpl.class); + private static final long CHECK_TRANSMISSION_COMPLETION_INTERVAL_IN_MILLISECONDS = 2_000L; + + private final Peer thisNode; + private final IStateMachine stateMachine; + private final Lock stateMachineLock = new ReentrantLock(); + private final PipeConsensusPeerManager peerManager; + private final AtomicBoolean active; + private final AtomicBoolean isStarted; + private final String consensusGroupId; + private final ConsensusPipeManager consensusPipeManager; + private final ProgressIndexManager progressIndexManager; + private final IClientManager<TEndPoint, SyncPipeConsensusServiceClient> syncClientManager; + + private ProgressIndex cachedProgressIndex = MinimumProgressIndex.INSTANCE; + + public PipeConsensusServerImpl( + Peer thisNode, + IStateMachine stateMachine, + String storageDir, + List<Peer> configuration, + PipeConsensusConfig config, + ConsensusPipeManager consensusPipeManager, + IClientManager<TEndPoint, SyncPipeConsensusServiceClient> syncClientManager) + throws IOException { + this.thisNode = thisNode; + this.stateMachine = stateMachine; + this.peerManager = new PipeConsensusPeerManager(storageDir, configuration); + this.active = new AtomicBoolean(true); + this.isStarted = new AtomicBoolean(false); + this.consensusGroupId = thisNode.getGroupId().toString(); + this.consensusPipeManager = consensusPipeManager; + this.progressIndexManager = config.getPipe().getProgressIndexManager(); + this.syncClientManager = syncClientManager; + + if (configuration.isEmpty()) { + peerManager.recover(); + } else { + // create consensus pipes + List<Peer> deepCopyPeersWithoutSelf = + configuration.stream() + .filter(peer -> !peer.equals(thisNode)) + .collect(Collectors.toList()); + final List<Peer> successfulPipes = createConsensusPipes(deepCopyPeersWithoutSelf); + if (successfulPipes.size() < deepCopyPeersWithoutSelf.size()) { + // roll back + updateConsensusPipesStatus(successfulPipes, PipeStatus.DROPPED); + throw new IOException(String.format("%s cannot create all consensus pipes", thisNode)); + } + + // persist peers' info + try { + peerManager.persistAll(); + } catch (Exception e) { + // roll back + LOGGER.warn("{} cannot persist all peers", thisNode, e); + updateConsensusPipesStatus(successfulPipes, PipeStatus.DROPPED); + throw e; + } + } + } + + public synchronized void start(boolean startConsensusPipes) throws IOException { + stateMachine.start(); + + if (startConsensusPipes) { + // start all consensus pipes + final List<Peer> otherPeers = peerManager.getOtherPeers(thisNode); + final List<Peer> successfulPipes = + updateConsensusPipesStatus(new ArrayList<>(otherPeers), PipeStatus.RUNNING); + if (successfulPipes.size() < otherPeers.size()) { + // roll back + updateConsensusPipesStatus(successfulPipes, PipeStatus.STOPPED); + throw new IOException(String.format("%s cannot start all consensus pipes", thisNode)); + } + } + isStarted.set(true); + } + + public synchronized void stop() { + // stop all consensus pipes + final List<Peer> otherPeers = peerManager.getOtherPeers(thisNode); + final List<Peer> successfulPipes = + updateConsensusPipesStatus(new ArrayList<>(otherPeers), PipeStatus.STOPPED); + if (successfulPipes.size() < otherPeers.size()) { + // do not roll back, because it will stop anyway + LOGGER.warn("{} cannot stop all consensus pipes", thisNode); + } + + stateMachine.stop(); + isStarted.set(false); + } + + public synchronized void clear() throws IOException { + final List<Peer> otherPeers = peerManager.getOtherPeers(thisNode); + final List<Peer> successfulPipes = + updateConsensusPipesStatus(new ArrayList<>(otherPeers), PipeStatus.DROPPED); + if (successfulPipes.size() < otherPeers.size()) { + // do not roll back, because it will clear anyway + LOGGER.warn("{} cannot drop all consensus pipes", thisNode); + } + + peerManager.clear(); + stateMachine.stop(); + isStarted.set(false); + active.set(false); + } + + private List<Peer> createConsensusPipes(List<Peer> peers) { + return peers.stream() + .filter( + peer -> { + try { + if (!peers.equals(thisNode)) { + consensusPipeManager.createConsensusPipe(thisNode, peer); + } + return true; + } catch (Exception e) { + LOGGER.warn("{} cannot create consensus pipe between {} and {}", thisNode, peer, e); + return false; + } + }) + .collect(Collectors.toList()); + } + + private List<Peer> updateConsensusPipesStatus(List<Peer> peers, PipeStatus status) { + return peers.stream() + .filter( + peer -> { + try { + if (!peer.equals(thisNode)) { + consensusPipeManager.updateConsensusPipe( + new ConsensusPipeName(thisNode, peer), status); + } + return true; + } catch (Exception e) { + LOGGER.warn( + "{} cannot update consensus pipe between {} and {} to status {}", + thisNode, + peer, + status, + e); + return false; + } + }) + .collect(Collectors.toList()); + } + + public synchronized void checkConsensusPipe(Map<ConsensusPipeName, PipeStatus> existedPipes) { + final PipeStatus expectedStatus = isStarted.get() ? PipeStatus.RUNNING : PipeStatus.STOPPED; + final Map<ConsensusPipeName, Peer> expectedPipes = + peerManager.getOtherPeers(thisNode).stream() + .collect( + ImmutableMap.toImmutableMap( + peer -> new ConsensusPipeName(thisNode, peer), peer -> peer)); + + existedPipes.forEach( + (existedName, existedStatus) -> { + if (!expectedPipes.containsKey(existedName)) { + try { + LOGGER.warn("{} drop consensus pipe [{}]", consensusGroupId, existedName); + consensusPipeManager.updateConsensusPipe(existedName, PipeStatus.DROPPED); + } catch (Exception e) { + LOGGER.warn("{} cannot drop consensus pipe [{}]", consensusGroupId, existedName, e); + } + } else if (!expectedStatus.equals(existedStatus)) { + try { + LOGGER.warn( + "{} update consensus pipe [{}] to status {}", + consensusGroupId, + existedName, + expectedStatus); + consensusPipeManager.updateConsensusPipe(existedName, expectedStatus); + } catch (Exception e) { + LOGGER.warn( + "{} cannot update consensus pipe [{}] to status {}", + consensusGroupId, + existedName, + expectedStatus, + e); + } + } + }); + + expectedPipes.forEach( + (expectedName, expectedPeer) -> { + if (!existedPipes.containsKey(expectedName)) { + try { + LOGGER.warn( + "{} create and update consensus pipe [{}] to status {}", + consensusGroupId, + expectedName, + expectedStatus); + consensusPipeManager.createConsensusPipe(thisNode, expectedPeer); + consensusPipeManager.updateConsensusPipe(expectedName, expectedStatus); + } catch (Exception e) { + LOGGER.warn( + "{} cannot create and update consensus pipe [{}] to status {}", + consensusGroupId, + expectedName, + expectedStatus, + e); + } + } + }); + } + + public TSStatus write(IConsensusRequest request) { + try { + stateMachineLock.lock(); + if (request instanceof ComparableConsensusRequest) { + ((ComparableConsensusRequest) request) + .setProgressIndex(progressIndexManager.assignProgressIndex(thisNode.getGroupId())); + } + Review Comment: fixed -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
