xichen01 commented on code in PR #4988: URL: https://github.com/apache/ozone/pull/4988#discussion_r1411723842
########## hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/block/SCMDeletedBlockTransactionStatusManager.java: ########## @@ -0,0 +1,616 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.hdds.scm.block; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.CommandStatus; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto; +import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult; +import org.apache.hadoop.hdds.scm.command.CommandStatusReportHandler.DeleteBlockStatus; +import org.apache.hadoop.hdds.scm.container.ContainerID; +import org.apache.hadoop.hdds.scm.container.ContainerInfo; +import org.apache.hadoop.hdds.scm.container.ContainerManager; +import org.apache.hadoop.hdds.scm.container.ContainerReplica; +import org.apache.hadoop.hdds.scm.ha.SCMContext; +import org.apache.hadoop.hdds.server.events.EventHandler; +import org.apache.hadoop.hdds.server.events.EventPublisher; +import org.apache.hadoop.ozone.protocol.commands.SCMCommand; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.Lock; +import java.util.stream.Collectors; + +import static java.lang.Math.min; +import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus; +import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus.SENT; +import static org.apache.hadoop.hdds.scm.block.SCMDeletedBlockTransactionStatusManager.SCMDeleteBlocksCommandStatusManager.CmdStatus.TO_BE_SENT; + +/** + * This is a class to manage the status of DeletedBlockTransaction, + * the purpose of this class is to reduce the number of duplicate + * DeletedBlockTransaction sent to the DN. + */ +public class SCMDeletedBlockTransactionStatusManager + implements EventHandler<DeleteBlockStatus> { + public static final Logger LOG = + LoggerFactory.getLogger(SCMDeletedBlockTransactionStatusManager.class); + // Maps txId to set of DNs which are successful in committing the transaction + private final Map<Long, Set<UUID>> transactionToDNsCommitMap; + // Maps txId to its retry counts; + private final Map<Long, Integer> transactionToRetryCountMap; + // The access to DeletedBlocksTXTable is protected by + // DeletedBlockLogStateManager. + private final DeletedBlockLogStateManager deletedBlockLogStateManager; + private final ContainerManager containerManager; + private final ScmBlockDeletingServiceMetrics metrics; + private final SCMContext scmContext; + private final Lock lock; + private final long scmCommandTimeoutMs; + + /** + * Before the DeletedBlockTransaction is executed on DN and reported to + * SCM, it is managed by this {@link SCMDeleteBlocksCommandStatusManager}. + * After the DeletedBlocksTransaction in the DeleteBlocksCommand is + * committed on the SCM, it is managed by + * {@link SCMDeletedBlockTransactionStatusManager#transactionToDNsCommitMap} + */ + private final SCMDeleteBlocksCommandStatusManager + scmDeleteBlocksCommandStatusManager; + + public SCMDeletedBlockTransactionStatusManager( + DeletedBlockLogStateManager deletedBlockLogStateManager, + ContainerManager containerManager, SCMContext scmContext, + ScmBlockDeletingServiceMetrics metrics, + Lock lock, long scmCommandTimeoutMs) { + // maps transaction to dns which have committed it. + this.deletedBlockLogStateManager = deletedBlockLogStateManager; + this.metrics = metrics; + this.containerManager = containerManager; + this.scmContext = scmContext; + this.lock = lock; + this.scmCommandTimeoutMs = scmCommandTimeoutMs; + this.transactionToDNsCommitMap = new ConcurrentHashMap<>(); + this.transactionToRetryCountMap = new ConcurrentHashMap<>(); + this.scmDeleteBlocksCommandStatusManager = + new SCMDeleteBlocksCommandStatusManager(); + } + + /** + * A class that manages the status of a DeletedBlockTransaction based + * on DeleteBlocksCommand. + */ + protected static class SCMDeleteBlocksCommandStatusManager { + public static final Logger LOG = + LoggerFactory.getLogger(SCMDeleteBlocksCommandStatusManager.class); + private final Map<UUID, Map<Long, CmdStatusData>> scmCmdStatusRecord; + + private static final CmdStatus DEFAULT_STATUS = TO_BE_SENT; + private static final Set<CmdStatus> STATUSES_REQUIRING_TIMEOUT = + new HashSet<>(Arrays.asList(SENT)); + + public SCMDeleteBlocksCommandStatusManager() { + this.scmCmdStatusRecord = new ConcurrentHashMap<>(); + } + + /** + * Status of SCMDeleteBlocksCommand. + */ + public enum CmdStatus { + // The DeleteBlocksCommand has not yet been sent. + // This is the initial status of the command after it's created. + TO_BE_SENT, + // If the DeleteBlocksCommand has been sent but has not been executed + // completely by DN, the DeleteBlocksCommand's state will be SENT. + // Note that the state of SENT includes the following possibilities. + // - The command was sent but not received + // - The command was sent and received by the DN, + // and is waiting to be executed. + // - The Command sent and being executed by DN + SENT, + } + + protected static final class CmdStatusData { + private final UUID dnId; + private final long scmCmdId; + private final Set<Long> deletedBlocksTxIds; + private Instant updateTime; + private CmdStatus status; + + private CmdStatusData( + UUID dnId, long scmTxID, Set<Long> deletedBlocksTxIds) { + this.dnId = dnId; + this.scmCmdId = scmTxID; + this.deletedBlocksTxIds = deletedBlocksTxIds; + setStatus(DEFAULT_STATUS); + } + + public Set<Long> getDeletedBlocksTxIds() { + return Collections.unmodifiableSet(deletedBlocksTxIds); + } + + public UUID getDnId() { + return dnId; + } + + public long getScmCmdId() { + return scmCmdId; + } + + public CmdStatus getStatus() { + return status; + } + + public void setStatus(CmdStatus status) { + this.updateTime = Instant.now(); + this.status = status; + } + + public Instant getUpdateTime() { + return updateTime; + } + + @Override + public String toString() { + return "ScmTxStateMachine" + + "{dnId=" + dnId + + ", scmTxID=" + scmCmdId + + ", deletedBlocksTxIds=" + deletedBlocksTxIds + + ", updateTime=" + updateTime + + ", status=" + status + + '}'; + } + } + + protected static CmdStatusData createScmCmdStatusData( + UUID dnId, long scmCmdId, Set<Long> deletedBlocksTxIds) { + return new CmdStatusData(dnId, scmCmdId, deletedBlocksTxIds); + } + + protected void recordScmCommand(CmdStatusData statusData) { + LOG.debug("Record ScmCommand: {}", statusData); + scmCmdStatusRecord.computeIfAbsent(statusData.getDnId(), k -> + new ConcurrentHashMap<>()).put(statusData.getScmCmdId(), statusData); + } + + protected void onSent(UUID dnId, long scmCmdId) { + updateStatus(dnId, scmCmdId, CommandStatus.Status.PENDING); + } + + protected void onDatanodeDead(UUID dnId) { + LOG.info("Clean SCMCommand record for DN: {}", dnId); + scmCmdStatusRecord.remove(dnId); + } + + protected void updateStatusByDNCommandStatus(UUID dnId, long scmCmdId, + CommandStatus.Status newState) { + updateStatus(dnId, scmCmdId, newState); + } + + protected void cleanAllTimeoutSCMCommand(long timeoutMs) { + for (UUID dnId : scmCmdStatusRecord.keySet()) { + for (CmdStatus status : STATUSES_REQUIRING_TIMEOUT) { + removeTimeoutScmCommand( + dnId, getScmCommandIds(dnId, status), timeoutMs); + } + } + } + + public void cleanTimeoutSCMCommand(UUID dnId, long timeoutMs) { + for (CmdStatus status : STATUSES_REQUIRING_TIMEOUT) { + removeTimeoutScmCommand( + dnId, getScmCommandIds(dnId, status), timeoutMs); + } + } + + private Set<Long> getScmCommandIds(UUID dnId, CmdStatus status) { + Set<Long> scmCmdIds = new HashSet<>(); + Map<Long, CmdStatusData> record = scmCmdStatusRecord.get(dnId); + if (record == null) { + return scmCmdIds; + } + for (CmdStatusData statusData : record.values()) { + if (statusData.getStatus().equals(status)) { + scmCmdIds.add(statusData.getScmCmdId()); + } + } + return scmCmdIds; + } + + private Instant getUpdateTime(UUID dnId, long scmCmdId) { + Map<Long, CmdStatusData> record = scmCmdStatusRecord.get(dnId); + if (record == null || record.get(scmCmdId) == null) { + return null; + } + return record.get(scmCmdId).getUpdateTime(); + } + + private void updateStatus(UUID dnId, long scmCmdId, + CommandStatus.Status newStatus) { + Map<Long, CmdStatusData> recordForDn = scmCmdStatusRecord.get(dnId); + if (recordForDn == null) { + LOG.warn("Unknown Datanode: {} scmCmdId {} newStatus {}", + dnId, scmCmdId, newStatus); + return; + } + if (recordForDn.get(scmCmdId) == null) { + LOG.warn("Unknown SCM Command: {} Datanode {} newStatus {}", + scmCmdId, dnId, newStatus); + return; + } + + boolean changed = false; + CmdStatusData statusData = recordForDn.get(scmCmdId); + CmdStatus oldStatus = statusData.getStatus(); + switch (newStatus) { + case PENDING: + if (oldStatus == TO_BE_SENT || oldStatus == SENT) { Review Comment: If `oldStatus == SENT`, the update status(`SENT->SENT`) will only be used to refresh the `updateTime` to prevent the command from timing out. ```java public void setStatus(CmdStatus status) { this.updateTime = Instant.now(); this.status = status; } ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
