This is an automated email from the ASF dual-hosted git repository. JackieTien97 pushed a commit to branch ty/mpp-notification-deadline in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0f596846ce923511305003182b6419098ee6ff54 Author: JackieTien97 <[email protected]> AuthorDate: Fri Nov 20 16:56:08 2026 +0800 Bound MPP sink notifications by the query deadline --- .../impl/DataNodeInternalRPCServiceImpl.java | 3 + .../execution/exchange/MPPDataExchangeManager.java | 45 ++-- .../execution/exchange/MPPDataExchangeService.java | 9 +- .../execution/exchange/sink/SinkChannel.java | 208 ++++++++++++----- .../fragment/FragmentInstanceContext.java | 12 + .../fragment/FragmentInstanceManager.java | 2 + .../plan/planner/plan/FragmentInstance.java | 10 + .../scheduler/FragmentInstanceDispatcherImpl.java | 8 + .../exchange/MPPDataExchangeManagerTest.java | 14 +- .../exchange/SinkChannelDeadlineTest.java | 259 +++++++++++++++++++++ .../SinkChannelFailurePropagationTest.java | 17 +- .../execution/exchange/SinkChannelTest.java | 30 +-- .../fragment/FragmentInstanceContextTest.java | 18 ++ .../FragmentInstanceDispatcherLifecycleTest.java | 4 + .../plan/scheduler/QueryAttemptProtocolTest.java | 2 + .../apache/iotdb/commons/client/ClientManager.java | 15 +- .../iotdb/commons/client/ClientPoolFactory.java | 25 -- .../client/sync/MPPDataExchangeClientManager.java | 155 ++++++++++++ .../SyncDataNodeMPPDataExchangeServiceClient.java | 165 ++++++++++--- .../iotdb/commons/concurrent/ThreadName.java | 2 + .../sync/MPPDataExchangeClientManagerTest.java | 158 +++++++++++++ .../src/main/thrift/datanode.thrift | 1 + 22 files changed, 986 insertions(+), 176 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 43f2e412abd..a621c3927e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -565,6 +565,9 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface FragmentInstance.deserializeFrom( fragmentInstanceId, req.fragmentInstance.body)); fragmentInstance.attachUDFQueryScope(queryScope); + if (req.isSetQueryDeadlineMs()) { + fragmentInstance.setQueryDeadlineMs(req.getQueryDeadlineMs()); + } if (req.isSetQueryAttemptId() && req.isSetAdmissionToken()) { fragmentInstance.setQueryAttempt(req.getQueryAttemptId(), req.getAdmissionToken()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java index 61e776d7d9e..763aeb169a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -692,8 +693,7 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager { private final LocalMemoryManager localMemoryManager; private final Supplier<TsBlockSerde> tsBlockSerdeFactory; private final ExecutorService executorService; - private final IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> - mppDataExchangeServiceClientManager; + private final MPPDataExchangeClientManager mppDataExchangeServiceClientManager; private final Map<TFragmentInstanceId, Map<String, ISourceHandle>> sourceHandles; /** Each FI has only one ShuffleSinkHandle. */ @@ -705,8 +705,7 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager { LocalMemoryManager localMemoryManager, Supplier<TsBlockSerde> tsBlockSerdeFactory, ExecutorService executorService, - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> - mppDataExchangeServiceClientManager) { + MPPDataExchangeClientManager mppDataExchangeServiceClientManager) { this.localMemoryManager = Validate.notNull( localMemoryManager, @@ -855,23 +854,29 @@ public class MPPDataExchangeManager implements IMPPDataExchangeManager { localFragmentInstanceId); } - return new SinkChannel( - remoteEndpoint, - remoteFragmentInstanceId, - remotePlanNodeId, - localPlanNodeId, - localFragmentInstanceId, - localMemoryManager, - executorService, - tsBlockSerdeFactory.get(), - new ISinkChannelListenerImpl( + SinkChannel channel = + new SinkChannel( + remoteEndpoint, + remoteFragmentInstanceId, + remotePlanNodeId, + localPlanNodeId, localFragmentInstanceId, - instanceContext, - instanceContext::failed, - cnt, - hasChannelFailedOrAborted), - instanceContext.isHighestPriority(), - mppDataExchangeServiceClientManager); + localMemoryManager, + executorService, + tsBlockSerdeFactory.get(), + new ISinkChannelListenerImpl( + localFragmentInstanceId, + instanceContext, + instanceContext::failed, + cnt, + hasChannelFailedOrAborted), + instanceContext.isHighestPriority(), + instanceContext.getExchangeDeadlineMs(), + mppDataExchangeServiceClientManager); + instanceContext + .getStateMachine() + .addFinalStateChangeListener(state -> channel.stopNotifications()); + return channel; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java index 6afcd2a3a91..8221bee92e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java @@ -20,9 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.client.ClientPoolFactory; -import org.apache.iotdb.commons.client.IClientManager; -import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.IoTThreadFactory; import org.apache.iotdb.commons.concurrent.ThreadName; @@ -76,9 +74,7 @@ public class MPPDataExchangeService extends ThriftService implements MPPDataExch new LocalMemoryManager(), new TsBlockSerdeFactory(), executorService, - new IClientManager.Factory<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient>() - .createClientManager( - new ClientPoolFactory.SyncDataNodeMPPDataExchangeServiceClientPoolFactory())); + new MPPDataExchangeClientManager()); LOGGER.info(DataNodeQueryMessages.MPPDATAEXCHANGEMANAGER_INIT_SUCCESSFULLY); } @@ -153,6 +149,7 @@ public class MPPDataExchangeService extends ThriftService implements MPPDataExch public void stop() { super.stop(); executorService.shutdown(); + mppDataExchangeManager.getMppDataExchangeServiceClientManager().close(); } public static MPPDataExchangeService getInstance() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java index 4d592771ef2..f85aef03ed5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java @@ -20,8 +20,9 @@ package org.apache.iotdb.db.queryengine.execution.exchange.sink; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.exception.QueryTimeoutException; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.DataNodeQueryMessages; @@ -41,6 +42,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TNewDataBlockEvent; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ListenableFuture; import org.apache.thrift.TApplicationException; +import org.apache.thrift.TException; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.external.commons.lang3.Validate; import org.apache.tsfile.read.common.block.TsBlock; @@ -51,11 +53,15 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; @@ -99,6 +105,10 @@ public class SinkChannel implements ISinkChannel { private final SinkListener sinkListener; private final String threadName; private long retryIntervalInMs; + private final long queryDeadlineMs; + private final Set<Thread> runningNotifications = new HashSet<>(); + private volatile CompletableFuture<Void> deadlineAlarm; + private volatile boolean notificationsStopped; private static final int DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); @@ -111,8 +121,7 @@ public class SinkChannel implements ISinkChannel { // size for current TsBlock to reserve and free private long currentTsBlockSize; - private final IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> - mppDataExchangeServiceClientManager; + private final MPPDataExchangeClientManager mppDataExchangeServiceClientManager; @SuppressWarnings("squid:S3077") private volatile ListenableFuture<Void> blocked; @@ -157,8 +166,7 @@ public class SinkChannel implements ISinkChannel { ExecutorService executorService, TsBlockSerde serde, SinkListener sinkListener, - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> - mppDataExchangeServiceClientManager) { + MPPDataExchangeClientManager mppDataExchangeServiceClientManager) { this( remoteEndpoint, remoteFragmentInstanceId, @@ -170,6 +178,7 @@ public class SinkChannel implements ISinkChannel { serde, sinkListener, false, + Long.MAX_VALUE, mppDataExchangeServiceClientManager); } @@ -185,8 +194,8 @@ public class SinkChannel implements ISinkChannel { TsBlockSerde serde, SinkListener sinkListener, boolean isHighestPriority, - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> - mppDataExchangeServiceClientManager) { + long queryDeadlineMs, + MPPDataExchangeClientManager mppDataExchangeServiceClientManager) { this.remoteEndpoint = Validate.notNull( remoteEndpoint, @@ -224,6 +233,7 @@ public class SinkChannel implements ISinkChannel { sinkListener, DataNodeQueryMessages.EXCEPTION_SINKLISTENER_CAN_NOT_BE_NULL_DOT_32C9E7C0); this.isHighestPriority = isHighestPriority; + this.queryDeadlineMs = queryDeadlineMs < 0 ? Long.MAX_VALUE : queryDeadlineMs; this.mppDataExchangeServiceClientManager = mppDataExchangeServiceClientManager; this.retryIntervalInMs = DEFAULT_RETRY_INTERVAL_IN_MS; this.threadName = @@ -319,6 +329,7 @@ public class SinkChannel implements ISinkChannel { if (aborted || closed) { return false; } + stopNotifications(); sequenceIdToTsBlock.clear(); if (blocked != null) { bufferRetainedSizeInBytes -= localMemoryManager.getQueryPool().tryCancel(blocked); @@ -349,6 +360,7 @@ public class SinkChannel implements ISinkChannel { if (closed || aborted) { return false; } + stopNotifications(); sequenceIdToTsBlock.clear(); if (blocked != null) { bufferRetainedSizeInBytes -= localMemoryManager.getQueryPool().tryCancel(blocked); @@ -581,6 +593,130 @@ public class SinkChannel implements ISinkChannel { // endregion + @FunctionalInterface + private interface Notification { + void send(SyncDataNodeMPPDataExchangeServiceClient client) throws TException; + } + + private void checkNotificationActive() throws InterruptedException { + if (deadlineReached()) { + throw new QueryTimeoutException(); + } + if (notificationsStopped || Thread.currentThread().isInterrupted()) { + throw new InterruptedException(); + } + } + + private boolean deadlineReached() { + return System.currentTimeMillis() >= queryDeadlineMs + || (deadlineAlarm != null && deadlineAlarm.isCompletedExceptionally()); + } + + private void registerNotification() throws InterruptedException { + synchronized (runningNotifications) { + checkNotificationActive(); + if (queryDeadlineMs != Long.MAX_VALUE && deadlineAlarm == null) { + deadlineAlarm = new CompletableFuture<>(); + deadlineAlarm + .orTimeout( + MPPDataExchangeClientManager.remainingMillis(queryDeadlineMs), + TimeUnit.MILLISECONDS) + .whenComplete( + (unused, failure) -> { + if (failure != null) { + interruptNotifications(); + } + }); + } + runningNotifications.add(Thread.currentThread()); + } + } + + private void interruptNotifications() { + synchronized (runningNotifications) { + runningNotifications.forEach(Thread::interrupt); + } + } + + /** Wake blocking borrows, socket I/O and retry waits when the channel or FI terminates. */ + public boolean stopNotifications() { + synchronized (runningNotifications) { + if (notificationsStopped) { + return false; + } + notificationsStopped = true; + if (deadlineAlarm != null) { + deadlineAlarm.complete(null); + } + interruptNotifications(); + return true; + } + } + + private boolean sendNotification(Notification notification, int blockCount) { + if (notificationsStopped) { + return false; + } + try { + registerNotification(); + try { + sendWithRetry(notification, blockCount); + return true; + } finally { + synchronized (runningNotifications) { + runningNotifications.remove(Thread.currentThread()); + } + } + } catch (Exception e) { + if (stopNotifications()) { + if (deadlineReached()) { + sinkListener.onFailure(this, new QueryTimeoutException()); + } else { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + sinkListener.onFailure(this, e); + } + } + return false; + } + } + + private void sendWithRetry(Notification notification, int blockCount) throws Exception { + for (int attempt = 1; ; attempt++) { + checkNotificationActive(); + long startTime = System.nanoTime(); + try (SyncDataNodeMPPDataExchangeServiceClient client = + mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint, queryDeadlineMs)) { + checkNotificationActive(); + notification.send(client); + checkNotificationActive(); + return; + } catch (Exception e) { + checkNotificationActive(); + LOGGER.warn( + blockCount == 0 + ? DataNodeQueryMessages.FAILED_TO_SEND_END_OF_DATA_BLOCK_EVENT + : DataNodeQueryMessages.FAILED_TO_SEND_NEW_DATA_BLOCK_EVENT_ATTEMPT, + attempt, + e); + if (attempt == MAX_ATTEMPT_TIMES || (blockCount > 0 && containsTApplicationException(e))) { + throw e; + } + Thread.sleep( + Math.min( + retryIntervalInMs, MPPDataExchangeClientManager.remainingMillis(queryDeadlineMs))); + } finally { + if (blockCount > 0) { + DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost( + SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - startTime); + DATA_EXCHANGE_COUNT_METRIC_SET.recordDataBlockNum( + SEND_NEW_DATA_BLOCK_NUM_CALLER, blockCount); + } + } + } + } + // region ============ inner class ============ /** * Send a {@link org.apache.iotdb.mpp.rpc.thrift.TNewDataBlockEvent} to downstream fragment @@ -613,7 +749,6 @@ public class SinkChannel implements ISinkChannel { remoteFragmentInstanceId, remotePlanNodeId); } - int attempt = 0; TNewDataBlockEvent newDataBlockEvent = new TNewDataBlockEvent( remoteFragmentInstanceId, @@ -621,33 +756,8 @@ public class SinkChannel implements ISinkChannel { localFragmentInstanceId, startSequenceId, blockSizes); - while (attempt < MAX_ATTEMPT_TIMES) { - attempt += 1; - long startTime = System.nanoTime(); - try (SyncDataNodeMPPDataExchangeServiceClient client = - mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint)) { - client.onNewDataBlockEvent(newDataBlockEvent); - break; - } catch (Exception e) { - LOGGER.warn( - DataNodeQueryMessages.FAILED_TO_SEND_NEW_DATA_BLOCK_EVENT_ATTEMPT, attempt, e); - if (containsTApplicationException(e) || attempt == MAX_ATTEMPT_TIMES) { - sinkListener.onFailure(SinkChannel.this, e); - return; - } - try { - Thread.sleep(retryIntervalInMs); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - sinkListener.onFailure(SinkChannel.this, e); - } - } finally { - DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost( - SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - startTime); - DATA_EXCHANGE_COUNT_METRIC_SET.recordDataBlockNum( - SEND_NEW_DATA_BLOCK_NUM_CALLER, blockSizes.size()); - } - } + sendNotification( + client -> client.onNewDataBlockEvent(newDataBlockEvent), blockSizes.size()); } } } @@ -664,35 +774,21 @@ public class SinkChannel implements ISinkChannel { if (LOGGER.isDebugEnabled()) { LOGGER.debug(DataNodeQueryMessages.NOTIFY_NO_MORE_TSBLOCK); } - int attempt = 0; TEndOfDataBlockEvent endOfDataBlockEvent = new TEndOfDataBlockEvent( remoteFragmentInstanceId, remotePlanNodeId, localFragmentInstanceId, nextSequenceId - 1); - while (attempt < MAX_ATTEMPT_TIMES) { - attempt += 1; - try (SyncDataNodeMPPDataExchangeServiceClient client = - mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint)) { - client.onEndOfDataBlockEvent(endOfDataBlockEvent); - break; - } catch (Exception e) { - LOGGER.warn(DataNodeQueryMessages.FAILED_TO_SEND_END_OF_DATA_BLOCK_EVENT, attempt, e); - if (attempt == MAX_ATTEMPT_TIMES) { - LOGGER.warn(DataNodeQueryMessages.FAILED_TO_SEND_END_OF_DATA_BLOCK_EVENT_2, e); - sinkListener.onFailure(SinkChannel.this, e); - return; - } - try { - Thread.sleep(retryIntervalInMs); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - sinkListener.onFailure(SinkChannel.this, e); - } + if (!sendNotification(client -> client.onEndOfDataBlockEvent(endOfDataBlockEvent), 0)) { + return; + } + synchronized (SinkChannel.this) { + if (notificationsStopped) { + return; } + noMoreTsBlocks = true; } - noMoreTsBlocks = true; if (isFinished()) { invokeOnFinished(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 6d33ea28a60..ddca5b2ea20 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -181,6 +181,7 @@ public class FragmentInstanceContext extends QueryContext { // Outer query deadline (startTime + timeout) for IoTDBLocal UDF private long outerQueryDeadlineMs = -1L; + private long exchangeDeadlineMs = -1L; private QueryAttemptRegistry.Registration queryAttemptRegistration; private boolean executionInitialized; @@ -662,6 +663,17 @@ public class FragmentInstanceContext extends QueryContext { return outerQueryDeadlineMs; } + void setExchangeDeadlineMs(long coordinatorDeadlineMs) { + exchangeDeadlineMs = coordinatorDeadlineMs; + } + + public long getExchangeDeadlineMs() { + // Legacy coordinators only send the remaining FI timeout. Also honor an earlier local FI limit. + return exchangeDeadlineMs < 0 + ? outerQueryDeadlineMs + : Math.min(outerQueryDeadlineMs, exchangeDeadlineMs); + } + public Optional<Throwable> getFailureCause() { return Optional.ofNullable( stateMachine.getFailureCauses().stream() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java index 7926d2b501a..2f7b4d5ff5f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java @@ -180,6 +180,7 @@ public class FragmentInstanceManager { DriverTask.computeDeadlineTimeInMs(instance.getTimeOut()), instance.isDebug(), instance.isVerbose()); + createdContext.setExchangeDeadlineMs(instance.getQueryDeadlineMs()); createdContext.adoptUDFQueryScope(instance.claimUDFQueryScope()); return createdContext; } catch (Throwable t) { @@ -356,6 +357,7 @@ public class FragmentInstanceManager { DriverTask.computeDeadlineTimeInMs(instance.getTimeOut()), instance.isDebug(), instance.isVerbose()); + createdContext.setExchangeDeadlineMs(instance.getQueryDeadlineMs()); createdContext.adoptUDFQueryScope(instance.claimUDFQueryScope()); return createdContext; } catch (Throwable t) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java index 477a48ad56c..b07123bc19c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java @@ -65,6 +65,8 @@ public class FragmentInstance implements IConsensusRequest { private final TimePredicate globalTimePredicate; private final long timeOut; + // Carried in the optional dispatch envelope. Older coordinators only send the FI timeOut. + private long queryDeadlineMs = -1; private boolean isRoot; @@ -364,6 +366,14 @@ public class FragmentInstance implements IConsensusRequest { return hostDataNode; } + public void setQueryDeadlineMs(long queryDeadlineMs) { + this.queryDeadlineMs = queryDeadlineMs; + } + + public long getQueryDeadlineMs() { + return queryDeadlineMs; + } + public long getTimeOut() { return timeOut; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 35b8c70b328..1c6b05d482a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -90,6 +90,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; import java.util.stream.Collectors; +import static com.google.common.math.LongMath.saturatedAdd; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.apache.iotdb.calc.metric.QueryExecutionMetricSet.DISPATCH_READ; import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onThriftFrameOversizeException; @@ -508,6 +509,10 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { private void dispatchOneInstance(FragmentInstance instance) throws FragmentInstanceDispatchException, TException { checkNotAborted(); + if (queryContext.getTimeOut() > 0) { + instance.setQueryDeadlineMs( + saturatedAdd(queryContext.getStartTime(), queryContext.getTimeOut())); + } TEndPoint endPoint = instance.getHostDataNode().getInternalEndPoint(); if (isDispatchedToLocal(endPoint)) { prepareAdmission(instance, endPoint, null); @@ -539,6 +544,9 @@ public class FragmentInstanceDispatcherImpl implements IFragInstanceDispatcher { case READ: final TSendFragmentInstanceReq sendFragmentInstanceReq = new TSendFragmentInstanceReq(new TFragmentInstance(instance.serializeToByteBuffer())); + if (instance.getQueryDeadlineMs() >= 0) { + sendFragmentInstanceReq.setQueryDeadlineMs(instance.getQueryDeadlineMs()); + } prepareAdmission(instance, endPoint, client); if (instance.hasQueryAttempt()) { sendFragmentInstanceReq diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java index a5573159a94..b400ce92eac 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java @@ -20,9 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.client.ClientPoolFactory; -import org.apache.iotdb.commons.client.IClientManager; -import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; @@ -92,7 +90,7 @@ public class MPPDataExchangeManagerTest { localMemoryManager, new TsBlockSerdeFactory(), exchangeExecutor, - Mockito.mock(IClientManager.class)); + Mockito.mock(MPPDataExchangeClientManager.class)); ISourceHandle sourceHandle = exchangeManager.createSourceHandle( targetFragmentInstanceId, @@ -147,9 +145,7 @@ public class MPPDataExchangeManagerTest { mockLocalMemoryManager, new TsBlockSerdeFactory(), Executors.newSingleThreadExecutor(), - new IClientManager.Factory<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient>() - .createClientManager( - new ClientPoolFactory.SyncDataNodeMPPDataExchangeServiceClientPoolFactory())); + new MPPDataExchangeClientManager()); ISinkHandle shuffleSinkHandle = mppDataExchangeManager.createShuffleSinkHandle( @@ -211,9 +207,7 @@ public class MPPDataExchangeManagerTest { mockLocalMemoryManager, new TsBlockSerdeFactory(), Executors.newSingleThreadExecutor(), - new IClientManager.Factory<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient>() - .createClientManager( - new ClientPoolFactory.SyncDataNodeMPPDataExchangeServiceClientPoolFactory())); + new MPPDataExchangeClientManager()); ISourceHandle localSourceHandle = mppDataExchangeManager.createLocalSourceHandleForFragment( diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelDeadlineTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelDeadlineTest.java new file mode 100644 index 00000000000..9db213d2a94 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelDeadlineTest.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.exchange; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; +import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; +import org.apache.iotdb.commons.exception.QueryTimeoutException; +import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; +import org.apache.iotdb.db.queryengine.common.PlanFragmentId; +import org.apache.iotdb.db.queryengine.common.QueryId; +import org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SinkListener; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannelIndex; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannelLocation; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.ShuffleSinkHandle; +import org.apache.iotdb.db.queryengine.execution.exchange.sink.SinkChannel; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; +import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine; +import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager; +import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId; + +import org.apache.thrift.TException; +import org.junit.After; +import org.junit.Test; + +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isA; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class SinkChannelDeadlineTest { + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + private final SinkListener listener = mock(SinkListener.class); + private final LocalMemoryManager memory = mock(LocalMemoryManager.class); + + @After + public void tearDown() { + executor.shutdownNow(); + } + + private SinkChannel channel( + TEndPoint endpoint, long deadline, MPPDataExchangeClientManager clients) { + doReturn(Utils.createMockNonBlockedMemoryPool()).when(memory).getQueryPool(); + return new SinkChannel( + endpoint, + new TFragmentInstanceId("query", 0, "0"), + "exchange", + "sink", + new TFragmentInstanceId("query", 1, "0"), + memory, + executor, + Utils.createMockTsBlockSerde(1024), + listener, + false, + deadline, + clients); + } + + private void awaitWorker() throws Exception { + executor.submit(() -> {}).get(3, SECONDS); + } + + @Test + public void queuedEventsExpireWithoutBorrowingAndReleaseMemory() throws Exception { + MPPDataExchangeClientManager clients = mock(MPPDataExchangeClientManager.class); + CountDownLatch dequeue = new CountDownLatch(1); + executor.submit( + () -> { + dequeue.await(); + return null; + }); + long deadline = System.currentTimeMillis() + 200; + SinkChannel channel = channel(new TEndPoint("unused", 1), deadline, clients); + doAnswer(call -> channel.abort()) + .when(listener) + .onFailure(eq(channel), isA(QueryTimeoutException.class)); + channel.open(); + channel.send(Utils.createMockTsBlocks(1, 1024).get(0)); + channel.setNoMoreTsBlocks(); + assertFalse(dequeue.await(Math.max(0, deadline - System.currentTimeMillis()), MILLISECONDS)); + dequeue.countDown(); + awaitWorker(); + verify(clients, never()).borrowClient(any(), anyLong()); + verify(listener).onFailure(eq(channel), isA(QueryTimeoutException.class)); + verify(listener, never()).onEndOfBlocks(channel); + assertEquals(0, channel.getBufferRetainedSizeInBytes()); + } + + @Test + public void cancelledFragmentSkipsQueuedEndEventBeforeChannelCleanup() throws Exception { + MPPDataExchangeClientManager clients = mock(MPPDataExchangeClientManager.class); + CountDownLatch dequeue = new CountDownLatch(1); + executor.submit( + () -> { + dequeue.await(); + return null; + }); + doReturn(Utils.createMockNonBlockedMemoryPool()).when(memory).getQueryPool(); + FragmentInstanceId id = + new FragmentInstanceId(new PlanFragmentId(new QueryId("query"), 1), "0"); + FragmentInstanceStateMachine state = new FragmentInstanceStateMachine(id, Runnable::run); + FragmentInstanceContext context = + FragmentInstanceContext.createFragmentInstanceContext(id, state); + MPPDataExchangeManager exchange = + new MPPDataExchangeManager(memory, new TsBlockSerdeFactory(), executor, clients); + ISinkHandle sink = + exchange.createShuffleSinkHandle( + Collections.singletonList( + new DownStreamChannelLocation( + new TEndPoint("unused", 1), + new TFragmentInstanceId("query", 0, "0"), + "exchange")), + new DownStreamChannelIndex(0), + ShuffleSinkHandle.ShuffleStrategyEnum.PLAIN, + id.toThrift(), + "sink", + context); + SinkChannel channel = (SinkChannel) sink.getChannel(0); + channel.setNoMoreTsBlocks(); + state.cancel(); + dequeue.countDown(); + awaitWorker(); + verify(clients, never()).borrowClient(any(), anyLong()); + assertFalse(channel.isNoMoreTsBlocks()); + assertTrue(state.getFailureCauses().isEmpty()); + sink.abort(); + } + + @Test + public void retryBackoffSharesTheDeadline() throws Exception { + TEndPoint endpoint = new TEndPoint("unused", 1); + MPPDataExchangeClientManager clients = mock(MPPDataExchangeClientManager.class); + SyncDataNodeMPPDataExchangeServiceClient client = + mock(SyncDataNodeMPPDataExchangeServiceClient.class); + when(clients.borrowClient(eq(endpoint), anyLong())).thenReturn(client); + doThrow(new TException("transient failure")).when(client).onNewDataBlockEvent(any()); + SinkChannel channel = channel(endpoint, System.currentTimeMillis() + 800, clients); + channel.setRetryIntervalInMs(30_000); + channel.open(); + channel.send(Utils.createMockTsBlocks(1, 1024).get(0)); + awaitWorker(); + verify(client).onNewDataBlockEvent(any()); + verify(listener).onFailure(eq(channel), isA(QueryTimeoutException.class)); + channel.abort(); + } + + @Test + public void abortInterruptsPoolWaitWithoutDiscardingTheHeldConnection() throws Exception { + MPPDataExchangeClientManager clients = new MPPDataExchangeClientManager(); + clients.getPool().setMaxTotalPerKey(1); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + TEndPoint endpoint = + new TEndPoint(server.getInetAddress().getHostAddress(), server.getLocalPort()); + try (SyncDataNodeMPPDataExchangeServiceClient held = clients.borrowClient(endpoint); + Socket peer = server.accept()) { + SinkChannel channel = channel(endpoint, System.currentTimeMillis() + 20_000, clients); + channel.setNoMoreTsBlocks(); + long start = System.nanoTime(); + while (clients.getPool().getNumWaiters() == 0 + && System.nanoTime() - start < SECONDS.toNanos(3)) { + Thread.sleep(10); + } + assertEquals(1, clients.getPool().getNumWaiters()); + channel.abort(); + awaitWorker(); + verify(listener, never()).onFailure(any(), any()); + assertEquals(1, clients.getPool().getNumActive()); + } + assertEquals(1, clients.getPool().getNumIdle()); + } finally { + clients.close(); + } + } + + @Test + public void silentPeerReleasesWorkerAtDeadline() throws Exception { + verifyBlockedRead(false); + } + + @Test + public void abortInterruptsReadAndLeavesOtherConnectionUsable() throws Exception { + verifyBlockedRead(true); + } + + private void verifyBlockedRead(boolean abort) throws Exception { + MPPDataExchangeClientManager clients = new MPPDataExchangeClientManager(); + try (ServerSocket server = new ServerSocket(0, 2, InetAddress.getLoopbackAddress())) { + server.setSoTimeout(3_000); + TEndPoint endpoint = + new TEndPoint(server.getInetAddress().getHostAddress(), server.getLocalPort()); + // Warm up the connection and keep another idle connection to the same endpoint. + try (SyncDataNodeMPPDataExchangeServiceClient first = clients.borrowClient(endpoint); + SyncDataNodeMPPDataExchangeServiceClient second = clients.borrowClient(endpoint)) {} + long borrowed = clients.getPool().getBorrowedCount(); + // The pool is LIFO, and the first client was returned last. + try (Socket peer = server.accept(); + Socket other = server.accept()) { + peer.setSoTimeout(3_000); + long deadline = System.currentTimeMillis() + (abort ? 20_000 : 800); + SinkChannel channel = channel(endpoint, deadline, clients); + channel.setNoMoreTsBlocks(); + assertTrue(peer.getInputStream().read() >= 0); + if (abort) { + channel.abort(); + } + awaitWorker(); + assertEquals(borrowed + 1, clients.getPool().getBorrowedCount()); + assertEquals(0, clients.getPool().getNumActive()); + assertEquals(1, clients.getPool().getNumIdle()); + verify(listener, never()).onEndOfBlocks(channel); + if (abort) { + verify(listener, never()).onFailure(any(), any()); + } else { + verify(listener).onFailure(eq(channel), isA(QueryTimeoutException.class)); + } + } + } finally { + clients.close(); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java index 7030b5c210c..12e2cc105fe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; @@ -79,7 +80,7 @@ public class SinkChannelFailurePropagationTest { Mockito.mock(LocalMemoryManager.class), new TsBlockSerdeFactory(), exchangeExecutor, - Mockito.mock(IClientManager.class)); + Mockito.mock(MPPDataExchangeClientManager.class)); AtomicInteger remainingChannels = new AtomicInteger(1); AtomicBoolean hasChannelFailedOrAborted = new AtomicBoolean(false); AtomicBoolean callbackInvoked = new AtomicBoolean(false); @@ -131,8 +132,8 @@ public class SinkChannelFailurePropagationTest { MemoryPool memoryPool = Utils.createMockNonBlockedMemoryPool(); Mockito.when(localMemoryManager.getQueryPool()).thenReturn(memoryPool); - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> exchangeClientManager = - Mockito.mock(IClientManager.class); + MPPDataExchangeClientManager exchangeClientManager = + Mockito.mock(MPPDataExchangeClientManager.class); SyncDataNodeMPPDataExchangeServiceClient exchangeClient = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); TException expectedFailure = @@ -143,7 +144,8 @@ public class SinkChannelFailurePropagationTest { new TApplicationException( TApplicationException.INTERNAL_ERROR, "Internal error processing onNewDataBlockEvent"))); - Mockito.when(exchangeClientManager.borrowClient(secondEndpoint)).thenReturn(exchangeClient); + Mockito.when(exchangeClientManager.borrowClient(secondEndpoint, Long.MAX_VALUE)) + .thenReturn(exchangeClient); Mockito.doThrow(expectedFailure) .when(exchangeClient) .onNewDataBlockEvent(Mockito.any(TNewDataBlockEvent.class)); @@ -226,12 +228,13 @@ public class SinkChannelFailurePropagationTest { MemoryPool memoryPool = Utils.createMockNonBlockedMemoryPool(); Mockito.when(localMemoryManager.getQueryPool()).thenReturn(memoryPool); - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> exchangeClientManager = - Mockito.mock(IClientManager.class); + MPPDataExchangeClientManager exchangeClientManager = + Mockito.mock(MPPDataExchangeClientManager.class); SyncDataNodeMPPDataExchangeServiceClient exchangeClient = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); TException expectedFailure = new TException("injected onNewDataBlockEvent failure"); - Mockito.when(exchangeClientManager.borrowClient(exchangeEndpoint)).thenReturn(exchangeClient); + Mockito.when(exchangeClientManager.borrowClient(exchangeEndpoint, Long.MAX_VALUE)) + .thenReturn(exchangeClient); Mockito.doThrow(expectedFailure) .when(exchangeClient) .onNewDataBlockEvent(Mockito.any(TNewDataBlockEvent.class)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java index f22725c0fea..220993bbfad 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java @@ -20,8 +20,8 @@ package org.apache.iotdb.db.queryengine.execution.exchange; import org.apache.iotdb.common.rpc.thrift.TEndPoint; -import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager; import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; import org.apache.iotdb.commons.memory.MemoryManager; import org.apache.iotdb.db.conf.IoTDBDescriptor; @@ -67,13 +67,14 @@ public class SinkChannelTest { LocalMemoryManager mockLocalMemoryManager = Mockito.mock(LocalMemoryManager.class); MemoryPool mockMemoryPool = Utils.createMockNonBlockedMemoryPool(); Mockito.when(mockLocalMemoryManager.getQueryPool()).thenReturn(mockMemoryPool); - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> mockClientManager = - Mockito.mock(IClientManager.class); + MPPDataExchangeClientManager mockClientManager = + Mockito.mock(MPPDataExchangeClientManager.class); // Construct a mock client. SyncDataNodeMPPDataExchangeServiceClient mockClient = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); try { - Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient); + Mockito.when(mockClientManager.borrowClient(remoteEndpoint, Long.MAX_VALUE)) + .thenReturn(mockClient); Mockito.doNothing() .when(mockClient) .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class)); @@ -232,13 +233,14 @@ public class SinkChannelTest { SinkListener mockSinkListener = Mockito.mock(SinkListener.class); // Construct several mock TsBlock(s). List<TsBlock> mockTsBlocks = Utils.createMockTsBlocks(numOfMockTsBlock, mockTsBlockSize); - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> mockClientManager = - Mockito.mock(IClientManager.class); + MPPDataExchangeClientManager mockClientManager = + Mockito.mock(MPPDataExchangeClientManager.class); // Construct a mock client. SyncDataNodeMPPDataExchangeServiceClient mockClient = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); try { - Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient); + Mockito.when(mockClientManager.borrowClient(remoteEndpoint, Long.MAX_VALUE)) + .thenReturn(mockClient); Mockito.doNothing() .when(mockClient) .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class)); @@ -436,14 +438,15 @@ public class SinkChannelTest { SinkListener mockSinkListener = Mockito.mock(SinkListener.class); // Construct several mock TsBlock(s). List<TsBlock> mockTsBlocks = Utils.createMockTsBlocks(numOfMockTsBlock, mockTsBlockSize); - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> mockClientManager = - Mockito.mock(IClientManager.class); + MPPDataExchangeClientManager mockClientManager = + Mockito.mock(MPPDataExchangeClientManager.class); // Construct a mock client. SyncDataNodeMPPDataExchangeServiceClient mockClient = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); TException mockException = new TException("Mock exception"); try { - Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient); + Mockito.when(mockClientManager.borrowClient(remoteEndpoint, Long.MAX_VALUE)) + .thenReturn(mockClient); Mockito.doThrow(mockException) .when(mockClient) .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class)); @@ -547,13 +550,14 @@ public class SinkChannelTest { SinkListener mockSinkListener = Mockito.mock(SinkListener.class); // Construct several mock TsBlock(s). List<TsBlock> mockTsBlocks = Utils.createMockTsBlocks(numOfMockTsBlock, mockTsBlockSize); - IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> mockClientManager = - Mockito.mock(IClientManager.class); + MPPDataExchangeClientManager mockClientManager = + Mockito.mock(MPPDataExchangeClientManager.class); // Construct a mock client. SyncDataNodeMPPDataExchangeServiceClient mockClient = Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class); try { - Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient); + Mockito.when(mockClientManager.borrowClient(remoteEndpoint, Long.MAX_VALUE)) + .thenReturn(mockClient); Mockito.doNothing() .when(mockClient) .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class)); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java index 4c103afff0d..304b5592eda 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java @@ -34,6 +34,7 @@ import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.PlanFragmentId; +import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.execution.aggregation.TreeAggregator; import org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.ITimeRangeIterator; import org.apache.iotdb.db.queryengine.execution.driver.DriverContext; @@ -75,6 +76,23 @@ import static org.junit.Assert.assertTrue; public class FragmentInstanceContextTest { + @Test + public void exchangeDeadlineHonorsBothCoordinatorAndLegacyFragmentLimits() { + FragmentInstanceId id = + new FragmentInstanceId(new PlanFragmentId(new QueryId("deadline"), 0), "0"); + FragmentInstanceStateMachine state = new FragmentInstanceStateMachine(id, Runnable::run); + long localDeadline = System.currentTimeMillis() + 30_000; + FragmentInstanceContext context = + FragmentInstanceContext.createFragmentInstanceContext( + id, state, null, localDeadline, false, false); + assertEquals(localDeadline, context.getExchangeDeadlineMs()); + context.setExchangeDeadlineMs(localDeadline - 1000); + assertEquals(localDeadline - 1000, context.getExchangeDeadlineMs()); + assertEquals(localDeadline, context.getOuterQueryDeadlineMs()); + context.setExchangeDeadlineMs(Long.MAX_VALUE); + assertEquals(localDeadline, context.getExchangeDeadlineMs()); + } + @Test public void testConcurrentQueryDataSourceStatistics() throws InterruptedException { ExecutorService instanceNotificationExecutor = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java index 5846254faad..aeaad8b934b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java @@ -119,6 +119,8 @@ public class FragmentInstanceDispatcherLifecycleTest { public void transportRetryReusesTokenAndStopFreezesIssuedTokens() throws Exception { FragmentInstanceDispatcherImpl dispatcher = dispatcher(); context.prepareForRetry(); + context.setTimeOut(30_000); + context.setStartTime(System.currentTimeMillis() - 3_200); List<TSendFragmentInstanceReq> requests = new ArrayList<>(); when(client.sendFragmentInstance(any())) .thenAnswer( @@ -133,6 +135,8 @@ public class FragmentInstanceDispatcherLifecycleTest { FragmentInstance instance = instance(0); assertTrue(dispatcher.dispatch(null, Collections.singletonList(instance)).get().isSuccessful()); assertEquals(2, requests.size()); + assertEquals(context.getStartTime() + 30_000, requests.get(0).getQueryDeadlineMs()); + assertEquals(requests.get(0).getQueryDeadlineMs(), requests.get(1).getQueryDeadlineMs()); assertEquals(0, requests.get(0).getQueryAttemptId()); assertEquals(token, requests.get(0).getAdmissionToken()); assertEquals(requests.get(0).getAdmissionToken(), requests.get(1).getAdmissionToken()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java index d77cdc8e1a7..7cde933e89d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java @@ -49,6 +49,7 @@ public class QueryAttemptProtocolTest { assertEquals(original, received); assertFalse(received.isSetQueryAttemptId()); assertFalse(received.isSetAdmissionToken()); + assertFalse(received.isSetQueryDeadlineMs()); TCancelQueryReq cancel = new TCancelQueryReq("query", Collections.emptyList(), true); TCancelQueryReq receivedCancel = new TCancelQueryReq(); new TDeserializer().deserialize(receivedCancel, new TSerializer().serialize(cancel)); @@ -62,6 +63,7 @@ public class QueryAttemptProtocolTest { TFragmentInstance body = new TFragmentInstance(ByteBuffer.wrap(new byte[] {1, 2, 3})); TSendFragmentInstanceReq request = new TSendFragmentInstanceReq(body) + .setQueryDeadlineMs(123456789L) .setQueryAttemptId(2) .setAdmissionToken(new TQueryAdmissionToken("receiver", 123450L, 1000L)); byte[] serialized = new TSerializer().serialize(request); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java index 336658fd5dd..b320b43393a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java @@ -38,7 +38,7 @@ public class ClientManager<K, V> implements IClientManager<K, V> { private final GenericKeyedObjectPool<K, V> pool; - ClientManager(IClientPoolFactory<K, V> factory) { + protected ClientManager(IClientPoolFactory<K, V> factory) { pool = factory.createClientPool(this); } @@ -49,11 +49,22 @@ public class ClientManager<K, V> implements IClientManager<K, V> { @Override public V borrowClient(K node) throws ClientManagerException { + return borrowClientWithTimeout(node, Long.MAX_VALUE); + } + + protected V borrowClientWithTimeout(K node, long maxWaitMs) throws ClientManagerException { if (node == null) { throw new BorrowNullClientManagerException(); } try { - return pool.borrowObject(node); + if (maxWaitMs == Long.MAX_VALUE) { + return pool.borrowObject(node); + } + long poolWaitMs = pool.getMaxWaitDuration().toMillis(); + return pool.borrowObject(node, poolWaitMs < 0 ? maxWaitMs : Math.min(poolWaitMs, maxWaitMs)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new ClientManagerException(e); } catch (Exception e) { throw new ClientManagerException(e); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java index ce5c8366889..a3b8e9af7e7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java @@ -34,7 +34,6 @@ import org.apache.iotdb.commons.client.property.ThriftClientProperty.DefaultProp import org.apache.iotdb.commons.client.sync.SyncAINodeClient; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient; -import org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient; import org.apache.iotdb.commons.client.sync.SyncIoTConsensusV2ServiceClient; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.conf.CommonConfig; @@ -251,30 +250,6 @@ public class ClientPoolFactory { } } - public static class SyncDataNodeMPPDataExchangeServiceClientPoolFactory - implements IClientPoolFactory<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> { - - @Override - public GenericKeyedObjectPool<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> - createClientPool( - ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> manager) { - GenericKeyedObjectPool<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> clientPool = - new GenericKeyedObjectPool<>( - new SyncDataNodeMPPDataExchangeServiceClient.Factory( - manager, - new ThriftClientProperty.Builder() - .setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS()) - .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) - .build()), - new ClientPoolProperty.Builder<SyncDataNodeMPPDataExchangeServiceClient>() - .build() - .getConfig()); - ClientManagerMetrics.getInstance() - .registerClientManager(this.getClass().getSimpleName(), clientPool); - return clientPool; - } - } - public static class AsyncDataNodeMPPDataExchangeServiceClientPoolFactory implements IClientPoolFactory<TEndPoint, AsyncDataNodeMPPDataExchangeServiceClient> { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManager.java new file mode 100644 index 00000000000..58bbf7494d3 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManager.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.client.sync; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.ClientManager; +import org.apache.iotdb.commons.client.ClientManagerMetrics; +import org.apache.iotdb.commons.client.IClientPoolFactory; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.property.ClientPoolProperty; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.concurrent.IoTDBDaemonThreadFactory; +import org.apache.iotdb.commons.concurrent.ThreadName; +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.exception.QueryTimeoutException; + +import org.apache.commons.pool2.impl.GenericKeyedObjectPool; +import org.apache.thrift.transport.TTransportException; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +public class MPPDataExchangeClientManager + extends ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> { + + // Native name resolution and TLS store loading can ignore interrupts. Keep these cold-path + // operations off exchange workers. Each waiting borrower removes its task on cancellation. + private final ThreadPoolExecutor connectionSetup = + new ThreadPoolExecutor( + 4, + 4, + 1, + TimeUnit.MINUTES, + new LinkedBlockingQueue<>(), + new IoTDBDaemonThreadFactory(ThreadName.MPP_DATA_EXCHANGE_CONNECTION_SETUP.getName())); + + public MPPDataExchangeClientManager() { + super(new SyncDataNodeMPPDataExchangeServiceClientPoolFactory()); + connectionSetup.allowCoreThreadTimeOut(true); + } + + @Override + public SyncDataNodeMPPDataExchangeServiceClient borrowClient(TEndPoint endpoint) + throws ClientManagerException { + return borrowClient(endpoint, Long.MAX_VALUE); + } + + public SyncDataNodeMPPDataExchangeServiceClient borrowClient(TEndPoint endpoint, long deadlineMs) + throws ClientManagerException { + SyncDataNodeMPPDataExchangeServiceClient client = + borrowClientWithTimeout(endpoint, remainingMillis(deadlineMs)); + boolean ready = false; + try { + client.prepare(deadlineMs); + ready = true; + return client; + } catch (Exception e) { + throw new ClientManagerException(e); + } finally { + if (!ready) { + try { + client.invalidate(); + } finally { + client.close(); + } + } + } + } + + <T> T prepareConnection(Callable<T> prepare, int timeoutMs) throws TTransportException { + FutureTask<T> task = new FutureTask<>(prepare); + connectionSetup.execute(task); + try { + return timeoutMs == 0 ? task.get() : task.get(timeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new TTransportException(e); + } catch (TimeoutException e) { + throw new TTransportException(TTransportException.TIMED_OUT, e); + } catch (ExecutionException e) { + throw new TTransportException(e.getCause()); + } finally { + if (task.cancel(true)) { + connectionSetup.remove(task); + } + } + } + + @Override + public void close() { + for (Runnable queued : connectionSetup.shutdownNow()) { + ((FutureTask<?>) queued).cancel(false); + } + super.close(); + } + + public static long remainingMillis(long deadlineMs) { + if (deadlineMs == Long.MAX_VALUE) { + return Long.MAX_VALUE; + } + long remaining = deadlineMs - System.currentTimeMillis(); + if (remaining <= 0) { + throw new QueryTimeoutException(); + } + return remaining; + } + + private static class SyncDataNodeMPPDataExchangeServiceClientPoolFactory + implements IClientPoolFactory<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> { + + @Override + public GenericKeyedObjectPool<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> + createClientPool( + ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> manager) { + CommonConfig conf = CommonDescriptor.getInstance().getConfig(); + GenericKeyedObjectPool<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> clientPool = + new GenericKeyedObjectPool<>( + new SyncDataNodeMPPDataExchangeServiceClient.Factory( + manager, + new ThriftClientProperty.Builder() + .setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS()) + .setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled()) + .build()), + new ClientPoolProperty.Builder<SyncDataNodeMPPDataExchangeServiceClient>() + .build() + .getConfig()); + ClientManagerMetrics.getInstance() + .registerClientManager(this.getClass().getSimpleName(), clientPool); + return clientPool; + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java index b4393bfd4ee..48beeab7c92 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java @@ -26,57 +26,128 @@ import org.apache.iotdb.commons.client.factory.ThriftClientFactory; import org.apache.iotdb.commons.client.property.ThriftClientProperty; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.i18n.ServiceMessages; import org.apache.iotdb.mpp.rpc.thrift.MPPDataExchangeService; import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory; -import org.apache.iotdb.rpc.TConfigurationConst; +import org.apache.iotdb.rpc.RpcSslUtils; import org.apache.iotdb.rpc.TimeoutChangeableTransport; +import com.google.common.net.InetAddresses; import org.apache.commons.pool2.PooledObject; import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransport; import org.apache.thrift.transport.TTransportException; +import org.apache.tsfile.external.commons.lang3.StringUtils; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; import java.net.SocketException; +import java.nio.channels.SocketChannel; +import java.security.GeneralSecurityException; public class SyncDataNodeMPPDataExchangeServiceClient extends MPPDataExchangeService.Client implements ThriftClient, AutoCloseable { - private final boolean printLogWhenEncounterException; - private final TEndPoint endpoint; - private final ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> clientManager; private static final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig(); - public SyncDataNodeMPPDataExchangeServiceClient( + private final ThriftClientProperty property; + private final TEndPoint endpoint; + private final MPPDataExchangeClientManager clientManager; + private boolean invalidated; + private boolean restoreTimeout; + + // The pool allocates the client before connecting so a stalled connect cannot block other + // borrowers inside Commons Pool's per-key makeObjectCountLock. + protected SyncDataNodeMPPDataExchangeServiceClient( ThriftClientProperty property, TEndPoint endpoint, - ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> clientManager) - throws TTransportException { - super( - property - .getProtocolFactory() - .getProtocol( - commonConfig.isEnableInternalSSL() - ? DeepCopyRpcTransportFactory.INSTANCE.getTransport( - endpoint.getIp(), - endpoint.getPort(), - property.getConnectionTimeoutMs(), - commonConfig.getTrustStorePath(), - commonConfig.getTrustStorePwd(), - commonConfig.getKeyStorePath(), - commonConfig.getKeyStorePwd(), - commonConfig.getSslProtocol()) - : DeepCopyRpcTransportFactory.INSTANCE.getTransport( - new TSocket( - TConfigurationConst.defaultTConfiguration, - endpoint.getIp(), - endpoint.getPort(), - property.getConnectionTimeoutMs())))); - this.printLogWhenEncounterException = property.isPrintLogWhenEncounterException(); + MPPDataExchangeClientManager clientManager) { + super(null); + this.property = property; this.endpoint = endpoint; this.clientManager = clientManager; - if (!getInputProtocol().getTransport().isOpen()) { - getInputProtocol().getTransport().open(); + } + + final void prepare(long deadlineMs) throws TTransportException, IOException { + if (iprot_ == null) { + // A channel-backed socket makes blocking connects, writes and reads interruptible, + // including reads during the TLS handshake. + TTransport transport = connect(SocketChannel.open().socket(), deadlineMs); + try { + iprot_ = oprot_ = property.getProtocolFactory().getProtocol(transport); + } catch (RuntimeException | Error e) { + transport.close(); + throw e; + } + } + if (deadlineMs != Long.MAX_VALUE) { + setTimeout(rpcTimeout(deadlineMs)); + } + } + + private record Peer(InetSocketAddress address, SSLSocketFactory sslFactory) {} + + private Peer resolvePeer(long deadlineMs) throws TTransportException { + String host = endpoint.getIp(); + int port = endpoint.getPort(); + boolean ssl = commonConfig.isEnableInternalSSL(); + if (!ssl && InetAddresses.isInetAddress(host)) { + return new Peer(new InetSocketAddress(InetAddresses.forString(host), port), null); + } + return clientManager.prepareConnection( + () -> new Peer(new InetSocketAddress(host, port), ssl ? sslSocketFactory() : null), + rpcTimeout(deadlineMs)); + } + + final TTransport connect(Socket socket, long deadlineMs) throws IOException, TTransportException { + try { + Peer peer = resolvePeer(deadlineMs); + socket.connect(peer.address(), rpcTimeout(deadlineMs)); + if (peer.sslFactory() != null) { + SSLSocket ssl = + (SSLSocket) + peer.sslFactory().createSocket(socket, endpoint.getIp(), endpoint.getPort(), true); + socket = ssl; + SSLParameters params = ssl.getSSLParameters(); + params.setEndpointIdentificationAlgorithm("HTTPS"); + ssl.setSSLParameters(params); + } + socket.setSoTimeout(rpcTimeout(deadlineMs)); + return DeepCopyRpcTransportFactory.INSTANCE.getTransport(new TSocket(socket)); + } catch (IOException | TTransportException | RuntimeException | Error e) { + socket.close(); + throw e; + } + } + + private static SSLSocketFactory sslSocketFactory() + throws GeneralSecurityException, IOException, TTransportException { + if (StringUtils.isBlank(commonConfig.getKeyStorePath()) + && StringUtils.isBlank(commonConfig.getTrustStorePath())) { + throw new TTransportException(ServiceMessages.KEYSTORE_OR_TRUSTSTORE_NOT_FOUND); + } + return RpcSslUtils.createSSLContext( + commonConfig.getKeyStorePath(), + commonConfig.getKeyStorePwd(), + commonConfig.getTrustStorePath(), + commonConfig.getTrustStorePwd(), + commonConfig.getSslProtocol()) + .getSocketFactory(); + } + + private int rpcTimeout(long deadlineMs) { + int configured = property.getConnectionTimeoutMs(); + if (deadlineMs == Long.MAX_VALUE) { + return configured; } + long remaining = MPPDataExchangeClientManager.remainingMillis(deadlineMs); + return (int) Math.min(remaining, configured > 0 ? configured : Integer.MAX_VALUE); } public int getTimeout() throws SocketException { @@ -84,6 +155,7 @@ public class SyncDataNodeMPPDataExchangeServiceClient extends MPPDataExchangeSer } public void setTimeout(int timeout) { + restoreTimeout = timeout != property.getConnectionTimeoutMs(); // the same transport is used in both input and output ((TimeoutChangeableTransport) (getInputProtocol().getTransport())).setTimeout(timeout); } @@ -95,17 +167,25 @@ public class SyncDataNodeMPPDataExchangeServiceClient extends MPPDataExchangeSer @Override public void invalidate() { - getInputProtocol().getTransport().close(); + if (!invalidated) { + invalidated = true; + if (iprot_ != null) { + iprot_.getTransport().close(); + } + } } @Override public void invalidateAll() { - clientManager.clear(endpoint); + // An interrupt closes just this channel. Other clients to the same peer remain usable. + if (!Thread.currentThread().isInterrupted()) { + clientManager.clear(endpoint); + } } @Override public boolean printLogWhenEncounterException() { - return printLogWhenEncounterException; + return property.isPrintLogWhenEncounterException(); } @Override @@ -113,7 +193,7 @@ public class SyncDataNodeMPPDataExchangeServiceClient extends MPPDataExchangeSer return String.format("SyncDataNodeMPPDataExchangeServiceClient{%s}", endpoint); } - public static class Factory + static class Factory extends ThriftClientFactory<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> { public Factory( @@ -134,17 +214,28 @@ public class SyncDataNodeMPPDataExchangeServiceClient extends MPPDataExchangeSer return new DefaultPooledObject<>( SyncThriftClientWithErrorHandler.newErrorHandler( SyncDataNodeMPPDataExchangeServiceClient.class, - SyncDataNodeMPPDataExchangeServiceClient.class.getConstructor( - thriftClientProperty.getClass(), endpoint.getClass(), clientManager.getClass()), + SyncDataNodeMPPDataExchangeServiceClient.class.getDeclaredConstructor( + ThriftClientProperty.class, TEndPoint.class, MPPDataExchangeClientManager.class), thriftClientProperty, endpoint, clientManager)); } + @Override + public void passivateObject( + TEndPoint endpoint, PooledObject<SyncDataNodeMPPDataExchangeServiceClient> pooledObject) { + SyncDataNodeMPPDataExchangeServiceClient client = pooledObject.getObject(); + if (client.restoreTimeout) { + client.setTimeout(thriftClientProperty.getConnectionTimeoutMs()); + } + } + @Override public boolean validateObject( TEndPoint endpoint, PooledObject<SyncDataNodeMPPDataExchangeServiceClient> pooledObject) { - return pooledObject.getObject().getInputProtocol().getTransport().isOpen(); + SyncDataNodeMPPDataExchangeServiceClient client = pooledObject.getObject(); + return !client.invalidated + && (client.iprot_ == null || client.iprot_.getTransport().isOpen()); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java index f0d3b02dcee..cc27a56171f 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java @@ -41,6 +41,7 @@ public enum ThreadName { // -------------------------- MPP -------------------------- MPP_COORDINATOR_SCHEDULED_EXECUTOR("MPP-Coordinator-Scheduled-Executor"), MPP_DATA_EXCHANGE_TASK_EXECUTOR("MPP-Data-Exchange-Task-Executors"), + MPP_DATA_EXCHANGE_CONNECTION_SETUP("MPP-Data-Exchange-Connection-Setup"), ASYNC_DATANODE_CLIENT_POOL("AsyncDataNodeInternalServiceClientPool"), MPP_DATA_EXCHANGE_RPC_SERVICE("MPPDataExchangeRPC-Service"), MPP_DATA_EXCHANGE_RPC_PROCESSOR("MPPDataExchangeRPC-Processor"), @@ -232,6 +233,7 @@ public enum ThreadName { Arrays.asList( MPP_COORDINATOR_SCHEDULED_EXECUTOR, MPP_DATA_EXCHANGE_TASK_EXECUTOR, + MPP_DATA_EXCHANGE_CONNECTION_SETUP, ASYNC_DATANODE_CLIENT_POOL, MPP_DATA_EXCHANGE_RPC_SERVICE, MPP_DATA_EXCHANGE_RPC_PROCESSOR, diff --git a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManagerTest.java b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManagerTest.java new file mode 100644 index 00000000000..dbd1b76352f --- /dev/null +++ b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManagerTest.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.client.sync; + +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.client.property.ThriftClientProperty; +import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.exception.QueryTimeoutException; + +import org.apache.thrift.transport.TTransportException; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketTimeoutException; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class MPPDataExchangeClientManagerTest { + @Test + public void reusedConnectionUsesEachBorrowersTimeout() throws Exception { + MPPDataExchangeClientManager manager = new MPPDataExchangeClientManager(); + int configured = CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS(); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + TEndPoint endpoint = + new TEndPoint(server.getInetAddress().getHostAddress(), server.getLocalPort()); + SyncDataNodeMPPDataExchangeServiceClient first; + try (SyncDataNodeMPPDataExchangeServiceClient client = manager.borrowClient(endpoint)) { + first = client; + assertEquals(configured, client.getTimeout()); + } + try (Socket peer = server.accept()) { + try (SyncDataNodeMPPDataExchangeServiceClient shortQuery = + manager.borrowClient(endpoint, System.currentTimeMillis() + 800)) { + assertSame(first, shortQuery); + assertTrue(shortQuery.getTimeout() > 0 && shortQuery.getTimeout() <= 800); + } + try (SyncDataNodeMPPDataExchangeServiceClient legacy = manager.borrowClient(endpoint)) { + assertSame(first, legacy); + assertEquals(configured, legacy.getTimeout()); + } + try (SyncDataNodeMPPDataExchangeServiceClient longQuery = + manager.borrowClient(endpoint, System.currentTimeMillis() + 20_000)) { + assertSame(first, longQuery); + assertTrue(longQuery.getTimeout() > 800 && longQuery.getTimeout() <= 20_000); + } + try (SyncDataNodeMPPDataExchangeServiceClient capped = + manager.borrowClient(endpoint, System.currentTimeMillis() + 2L * configured)) { + assertEquals(configured, capped.getTimeout()); + } + } + } finally { + manager.close(); + } + } + + @Test(timeout = 10_000) + public void exhaustedPoolHonorsBothQueryAndPoolWaitLimits() throws Exception { + MPPDataExchangeClientManager manager = new MPPDataExchangeClientManager(); + manager.getPool().setMaxTotalPerKey(1); + try (ServerSocket server = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + TEndPoint endpoint = + new TEndPoint(server.getInetAddress().getHostAddress(), server.getLocalPort()); + try (SyncDataNodeMPPDataExchangeServiceClient held = manager.borrowClient(endpoint); + Socket peer = server.accept()) { + long deadline = System.currentTimeMillis() + 150; + assertThrows(ClientManagerException.class, () -> manager.borrowClient(endpoint, deadline)); + assertTrue(System.currentTimeMillis() >= deadline - 1); + manager.getPool().setMaxWait(Duration.ofMillis(30)); + assertThrows( + ClientManagerException.class, + () -> manager.borrowClient(endpoint, System.currentTimeMillis() + 20_000)); + assertEquals(1, manager.getPool().getNumActive()); + } + assertEquals(1, manager.getPool().getNumIdle()); + } finally { + manager.close(); + } + } + + @Test(timeout = 5_000) + public void stalledConnectionSetupReleasesBorrower() throws Exception { + MPPDataExchangeClientManager manager = new MPPDataExchangeClientManager(); + CountDownLatch release = new CountDownLatch(1); + try { + TTransportException failure = + assertThrows( + TTransportException.class, + () -> + manager.prepareConnection( + () -> { + release.await(); + return null; + }, + 80)); + assertEquals(TTransportException.TIMED_OUT, failure.getType()); + assertEquals("ready", manager.prepareConnection(() -> "ready", 1_000)); + } finally { + release.countDown(); + manager.close(); + } + } + + @Test + public void connectUsesRemainingBudgetAndClosesFailedSocket() throws Exception { + SyncDataNodeMPPDataExchangeServiceClient client = + new SyncDataNodeMPPDataExchangeServiceClient( + new ThriftClientProperty.Builder().setConnectionTimeoutMs(60_000).build(), + new TEndPoint("127.0.0.1", 10740), + null); + Socket socket = mock(Socket.class); + doThrow(new SocketTimeoutException()).when(socket).connect(any(SocketAddress.class), anyInt()); + long deadline = System.currentTimeMillis() - 3_200 + 4_000; + assertThrows(SocketTimeoutException.class, () -> client.connect(socket, deadline)); + ArgumentCaptor<Integer> timeout = ArgumentCaptor.forClass(Integer.class); + verify(socket).connect(any(SocketAddress.class), timeout.capture()); + assertTrue(timeout.getValue() > 0 && timeout.getValue() <= 800); + verify(socket).close(); + + Socket expired = mock(Socket.class); + assertThrows( + QueryTimeoutException.class, () -> client.connect(expired, System.currentTimeMillis() - 1)); + verify(expired, never()).connect(any(SocketAddress.class), anyInt()); + verify(expired).close(); + } +} diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 8e2335fab0c..768081aad76 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -152,6 +152,7 @@ struct TSendFragmentInstanceReq { // Attempt metadata is absent for a legacy coordinator. Do not change the binary FI body. 3: optional i32 queryAttemptId 4: optional TQueryAdmissionToken admissionToken + 5: optional i64 queryDeadlineMs } struct TSendFragmentInstanceResp {
