This is an automated email from the ASF dual-hosted git repository.

JackieTien97 pushed a commit to branch ty/mpp-notification-deadline
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 0f596846ce923511305003182b6419098ee6ff54
Author: JackieTien97 <[email protected]>
AuthorDate: Fri Nov 20 16:56:08 2026 +0800

    Bound MPP sink notifications by the query deadline
---
 .../impl/DataNodeInternalRPCServiceImpl.java       |   3 +
 .../execution/exchange/MPPDataExchangeManager.java |  45 ++--
 .../execution/exchange/MPPDataExchangeService.java |   9 +-
 .../execution/exchange/sink/SinkChannel.java       | 208 ++++++++++++-----
 .../fragment/FragmentInstanceContext.java          |  12 +
 .../fragment/FragmentInstanceManager.java          |   2 +
 .../plan/planner/plan/FragmentInstance.java        |  10 +
 .../scheduler/FragmentInstanceDispatcherImpl.java  |   8 +
 .../exchange/MPPDataExchangeManagerTest.java       |  14 +-
 .../exchange/SinkChannelDeadlineTest.java          | 259 +++++++++++++++++++++
 .../SinkChannelFailurePropagationTest.java         |  17 +-
 .../execution/exchange/SinkChannelTest.java        |  30 +--
 .../fragment/FragmentInstanceContextTest.java      |  18 ++
 .../FragmentInstanceDispatcherLifecycleTest.java   |   4 +
 .../plan/scheduler/QueryAttemptProtocolTest.java   |   2 +
 .../apache/iotdb/commons/client/ClientManager.java |  15 +-
 .../iotdb/commons/client/ClientPoolFactory.java    |  25 --
 .../client/sync/MPPDataExchangeClientManager.java  | 155 ++++++++++++
 .../SyncDataNodeMPPDataExchangeServiceClient.java  | 165 ++++++++++---
 .../iotdb/commons/concurrent/ThreadName.java       |   2 +
 .../sync/MPPDataExchangeClientManagerTest.java     | 158 +++++++++++++
 .../src/main/thrift/datanode.thrift                |   1 +
 22 files changed, 986 insertions(+), 176 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 43f2e412abd..a621c3927e5 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -565,6 +565,9 @@ public class DataNodeInternalRPCServiceImpl implements 
IDataNodeRPCService.Iface
                       FragmentInstance.deserializeFrom(
                           fragmentInstanceId, req.fragmentInstance.body));
       fragmentInstance.attachUDFQueryScope(queryScope);
+      if (req.isSetQueryDeadlineMs()) {
+        fragmentInstance.setQueryDeadlineMs(req.getQueryDeadlineMs());
+      }
       if (req.isSetQueryAttemptId() && req.isSetAdmissionToken()) {
         fragmentInstance.setQueryAttempt(req.getQueryAttemptId(), 
req.getAdmissionToken());
       }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java
index 61e776d7d9e..763aeb169a5 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManager.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.execution.exchange;
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
 import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -692,8 +693,7 @@ public class MPPDataExchangeManager implements 
IMPPDataExchangeManager {
   private final LocalMemoryManager localMemoryManager;
   private final Supplier<TsBlockSerde> tsBlockSerdeFactory;
   private final ExecutorService executorService;
-  private final IClientManager<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>
-      mppDataExchangeServiceClientManager;
+  private final MPPDataExchangeClientManager 
mppDataExchangeServiceClientManager;
   private final Map<TFragmentInstanceId, Map<String, ISourceHandle>> 
sourceHandles;
 
   /** Each FI has only one ShuffleSinkHandle. */
@@ -705,8 +705,7 @@ public class MPPDataExchangeManager implements 
IMPPDataExchangeManager {
       LocalMemoryManager localMemoryManager,
       Supplier<TsBlockSerde> tsBlockSerdeFactory,
       ExecutorService executorService,
-      IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient>
-          mppDataExchangeServiceClientManager) {
+      MPPDataExchangeClientManager mppDataExchangeServiceClientManager) {
     this.localMemoryManager =
         Validate.notNull(
             localMemoryManager,
@@ -855,23 +854,29 @@ public class MPPDataExchangeManager implements 
IMPPDataExchangeManager {
           localFragmentInstanceId);
     }
 
-    return new SinkChannel(
-        remoteEndpoint,
-        remoteFragmentInstanceId,
-        remotePlanNodeId,
-        localPlanNodeId,
-        localFragmentInstanceId,
-        localMemoryManager,
-        executorService,
-        tsBlockSerdeFactory.get(),
-        new ISinkChannelListenerImpl(
+    SinkChannel channel =
+        new SinkChannel(
+            remoteEndpoint,
+            remoteFragmentInstanceId,
+            remotePlanNodeId,
+            localPlanNodeId,
             localFragmentInstanceId,
-            instanceContext,
-            instanceContext::failed,
-            cnt,
-            hasChannelFailedOrAborted),
-        instanceContext.isHighestPriority(),
-        mppDataExchangeServiceClientManager);
+            localMemoryManager,
+            executorService,
+            tsBlockSerdeFactory.get(),
+            new ISinkChannelListenerImpl(
+                localFragmentInstanceId,
+                instanceContext,
+                instanceContext::failed,
+                cnt,
+                hasChannelFailedOrAborted),
+            instanceContext.isHighestPriority(),
+            instanceContext.getExchangeDeadlineMs(),
+            mppDataExchangeServiceClientManager);
+    instanceContext
+        .getStateMachine()
+        .addFinalStateChangeListener(state -> channel.stopNotifications());
+    return channel;
   }
 
   @Override
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java
index 6afcd2a3a91..8221bee92e5 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeService.java
@@ -20,9 +20,7 @@
 package org.apache.iotdb.db.queryengine.execution.exchange;
 
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.commons.client.ClientPoolFactory;
-import org.apache.iotdb.commons.client.IClientManager;
-import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
 import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
 import org.apache.iotdb.commons.concurrent.IoTThreadFactory;
 import org.apache.iotdb.commons.concurrent.ThreadName;
@@ -76,9 +74,7 @@ public class MPPDataExchangeService extends ThriftService 
implements MPPDataExch
             new LocalMemoryManager(),
             new TsBlockSerdeFactory(),
             executorService,
-            new IClientManager.Factory<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>()
-                .createClientManager(
-                    new 
ClientPoolFactory.SyncDataNodeMPPDataExchangeServiceClientPoolFactory()));
+            new MPPDataExchangeClientManager());
     
LOGGER.info(DataNodeQueryMessages.MPPDATAEXCHANGEMANAGER_INIT_SUCCESSFULLY);
   }
 
@@ -153,6 +149,7 @@ public class MPPDataExchangeService extends ThriftService 
implements MPPDataExch
   public void stop() {
     super.stop();
     executorService.shutdown();
+    mppDataExchangeManager.getMppDataExchangeServiceClientManager().close();
   }
 
   public static MPPDataExchangeService getInstance() {
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
index 4d592771ef2..f85aef03ed5 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/exchange/sink/SinkChannel.java
@@ -20,8 +20,9 @@
 package org.apache.iotdb.db.queryengine.execution.exchange.sink;
 
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
 import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
+import org.apache.iotdb.commons.exception.QueryTimeoutException;
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
@@ -41,6 +42,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TNewDataBlockEvent;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.ListenableFuture;
 import org.apache.thrift.TApplicationException;
+import org.apache.thrift.TException;
 import org.apache.tsfile.common.conf.TSFileDescriptor;
 import org.apache.tsfile.external.commons.lang3.Validate;
 import org.apache.tsfile.read.common.block.TsBlock;
@@ -51,11 +53,15 @@ import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static com.google.common.util.concurrent.Futures.immediateVoidFuture;
@@ -99,6 +105,10 @@ public class SinkChannel implements ISinkChannel {
   private final SinkListener sinkListener;
   private final String threadName;
   private long retryIntervalInMs;
+  private final long queryDeadlineMs;
+  private final Set<Thread> runningNotifications = new HashSet<>();
+  private volatile CompletableFuture<Void> deadlineAlarm;
+  private volatile boolean notificationsStopped;
 
   private static final int DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES =
       TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes();
@@ -111,8 +121,7 @@ public class SinkChannel implements ISinkChannel {
   // size for current TsBlock to reserve and free
   private long currentTsBlockSize;
 
-  private final IClientManager<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>
-      mppDataExchangeServiceClientManager;
+  private final MPPDataExchangeClientManager 
mppDataExchangeServiceClientManager;
 
   @SuppressWarnings("squid:S3077")
   private volatile ListenableFuture<Void> blocked;
@@ -157,8 +166,7 @@ public class SinkChannel implements ISinkChannel {
       ExecutorService executorService,
       TsBlockSerde serde,
       SinkListener sinkListener,
-      IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient>
-          mppDataExchangeServiceClientManager) {
+      MPPDataExchangeClientManager mppDataExchangeServiceClientManager) {
     this(
         remoteEndpoint,
         remoteFragmentInstanceId,
@@ -170,6 +178,7 @@ public class SinkChannel implements ISinkChannel {
         serde,
         sinkListener,
         false,
+        Long.MAX_VALUE,
         mppDataExchangeServiceClientManager);
   }
 
@@ -185,8 +194,8 @@ public class SinkChannel implements ISinkChannel {
       TsBlockSerde serde,
       SinkListener sinkListener,
       boolean isHighestPriority,
-      IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient>
-          mppDataExchangeServiceClientManager) {
+      long queryDeadlineMs,
+      MPPDataExchangeClientManager mppDataExchangeServiceClientManager) {
     this.remoteEndpoint =
         Validate.notNull(
             remoteEndpoint,
@@ -224,6 +233,7 @@ public class SinkChannel implements ISinkChannel {
             sinkListener,
             
DataNodeQueryMessages.EXCEPTION_SINKLISTENER_CAN_NOT_BE_NULL_DOT_32C9E7C0);
     this.isHighestPriority = isHighestPriority;
+    this.queryDeadlineMs = queryDeadlineMs < 0 ? Long.MAX_VALUE : 
queryDeadlineMs;
     this.mppDataExchangeServiceClientManager = 
mppDataExchangeServiceClientManager;
     this.retryIntervalInMs = DEFAULT_RETRY_INTERVAL_IN_MS;
     this.threadName =
@@ -319,6 +329,7 @@ public class SinkChannel implements ISinkChannel {
     if (aborted || closed) {
       return false;
     }
+    stopNotifications();
     sequenceIdToTsBlock.clear();
     if (blocked != null) {
       bufferRetainedSizeInBytes -= 
localMemoryManager.getQueryPool().tryCancel(blocked);
@@ -349,6 +360,7 @@ public class SinkChannel implements ISinkChannel {
     if (closed || aborted) {
       return false;
     }
+    stopNotifications();
     sequenceIdToTsBlock.clear();
     if (blocked != null) {
       bufferRetainedSizeInBytes -= 
localMemoryManager.getQueryPool().tryCancel(blocked);
@@ -581,6 +593,130 @@ public class SinkChannel implements ISinkChannel {
 
   // endregion
 
+  @FunctionalInterface
+  private interface Notification {
+    void send(SyncDataNodeMPPDataExchangeServiceClient client) throws 
TException;
+  }
+
+  private void checkNotificationActive() throws InterruptedException {
+    if (deadlineReached()) {
+      throw new QueryTimeoutException();
+    }
+    if (notificationsStopped || Thread.currentThread().isInterrupted()) {
+      throw new InterruptedException();
+    }
+  }
+
+  private boolean deadlineReached() {
+    return System.currentTimeMillis() >= queryDeadlineMs
+        || (deadlineAlarm != null && deadlineAlarm.isCompletedExceptionally());
+  }
+
+  private void registerNotification() throws InterruptedException {
+    synchronized (runningNotifications) {
+      checkNotificationActive();
+      if (queryDeadlineMs != Long.MAX_VALUE && deadlineAlarm == null) {
+        deadlineAlarm = new CompletableFuture<>();
+        deadlineAlarm
+            .orTimeout(
+                MPPDataExchangeClientManager.remainingMillis(queryDeadlineMs),
+                TimeUnit.MILLISECONDS)
+            .whenComplete(
+                (unused, failure) -> {
+                  if (failure != null) {
+                    interruptNotifications();
+                  }
+                });
+      }
+      runningNotifications.add(Thread.currentThread());
+    }
+  }
+
+  private void interruptNotifications() {
+    synchronized (runningNotifications) {
+      runningNotifications.forEach(Thread::interrupt);
+    }
+  }
+
+  /** Wake blocking borrows, socket I/O and retry waits when the channel or FI 
terminates. */
+  public boolean stopNotifications() {
+    synchronized (runningNotifications) {
+      if (notificationsStopped) {
+        return false;
+      }
+      notificationsStopped = true;
+      if (deadlineAlarm != null) {
+        deadlineAlarm.complete(null);
+      }
+      interruptNotifications();
+      return true;
+    }
+  }
+
+  private boolean sendNotification(Notification notification, int blockCount) {
+    if (notificationsStopped) {
+      return false;
+    }
+    try {
+      registerNotification();
+      try {
+        sendWithRetry(notification, blockCount);
+        return true;
+      } finally {
+        synchronized (runningNotifications) {
+          runningNotifications.remove(Thread.currentThread());
+        }
+      }
+    } catch (Exception e) {
+      if (stopNotifications()) {
+        if (deadlineReached()) {
+          sinkListener.onFailure(this, new QueryTimeoutException());
+        } else {
+          if (e instanceof InterruptedException) {
+            Thread.currentThread().interrupt();
+          }
+          sinkListener.onFailure(this, e);
+        }
+      }
+      return false;
+    }
+  }
+
+  private void sendWithRetry(Notification notification, int blockCount) throws 
Exception {
+    for (int attempt = 1; ; attempt++) {
+      checkNotificationActive();
+      long startTime = System.nanoTime();
+      try (SyncDataNodeMPPDataExchangeServiceClient client =
+          mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint, 
queryDeadlineMs)) {
+        checkNotificationActive();
+        notification.send(client);
+        checkNotificationActive();
+        return;
+      } catch (Exception e) {
+        checkNotificationActive();
+        LOGGER.warn(
+            blockCount == 0
+                ? DataNodeQueryMessages.FAILED_TO_SEND_END_OF_DATA_BLOCK_EVENT
+                : 
DataNodeQueryMessages.FAILED_TO_SEND_NEW_DATA_BLOCK_EVENT_ATTEMPT,
+            attempt,
+            e);
+        if (attempt == MAX_ATTEMPT_TIMES || (blockCount > 0 && 
containsTApplicationException(e))) {
+          throw e;
+        }
+        Thread.sleep(
+            Math.min(
+                retryIntervalInMs, 
MPPDataExchangeClientManager.remainingMillis(queryDeadlineMs)));
+      } finally {
+        if (blockCount > 0) {
+          DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost(
+              SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - 
startTime);
+          DATA_EXCHANGE_COUNT_METRIC_SET.recordDataBlockNum(
+              SEND_NEW_DATA_BLOCK_NUM_CALLER, blockCount);
+        }
+      }
+    }
+  }
+
   // region ============ inner class ============
   /**
    * Send a {@link org.apache.iotdb.mpp.rpc.thrift.TNewDataBlockEvent} to 
downstream fragment
@@ -613,7 +749,6 @@ public class SinkChannel implements ISinkChannel {
               remoteFragmentInstanceId,
               remotePlanNodeId);
         }
-        int attempt = 0;
         TNewDataBlockEvent newDataBlockEvent =
             new TNewDataBlockEvent(
                 remoteFragmentInstanceId,
@@ -621,33 +756,8 @@ public class SinkChannel implements ISinkChannel {
                 localFragmentInstanceId,
                 startSequenceId,
                 blockSizes);
-        while (attempt < MAX_ATTEMPT_TIMES) {
-          attempt += 1;
-          long startTime = System.nanoTime();
-          try (SyncDataNodeMPPDataExchangeServiceClient client =
-              
mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint)) {
-            client.onNewDataBlockEvent(newDataBlockEvent);
-            break;
-          } catch (Exception e) {
-            LOGGER.warn(
-                
DataNodeQueryMessages.FAILED_TO_SEND_NEW_DATA_BLOCK_EVENT_ATTEMPT, attempt, e);
-            if (containsTApplicationException(e) || attempt == 
MAX_ATTEMPT_TIMES) {
-              sinkListener.onFailure(SinkChannel.this, e);
-              return;
-            }
-            try {
-              Thread.sleep(retryIntervalInMs);
-            } catch (InterruptedException ex) {
-              Thread.currentThread().interrupt();
-              sinkListener.onFailure(SinkChannel.this, e);
-            }
-          } finally {
-            DATA_EXCHANGE_COST_METRIC_SET.recordDataExchangeCost(
-                SEND_NEW_DATA_BLOCK_EVENT_TASK_CALLER, System.nanoTime() - 
startTime);
-            DATA_EXCHANGE_COUNT_METRIC_SET.recordDataBlockNum(
-                SEND_NEW_DATA_BLOCK_NUM_CALLER, blockSizes.size());
-          }
-        }
+        sendNotification(
+            client -> client.onNewDataBlockEvent(newDataBlockEvent), 
blockSizes.size());
       }
     }
   }
@@ -664,35 +774,21 @@ public class SinkChannel implements ISinkChannel {
         if (LOGGER.isDebugEnabled()) {
           LOGGER.debug(DataNodeQueryMessages.NOTIFY_NO_MORE_TSBLOCK);
         }
-        int attempt = 0;
         TEndOfDataBlockEvent endOfDataBlockEvent =
             new TEndOfDataBlockEvent(
                 remoteFragmentInstanceId,
                 remotePlanNodeId,
                 localFragmentInstanceId,
                 nextSequenceId - 1);
-        while (attempt < MAX_ATTEMPT_TIMES) {
-          attempt += 1;
-          try (SyncDataNodeMPPDataExchangeServiceClient client =
-              
mppDataExchangeServiceClientManager.borrowClient(remoteEndpoint)) {
-            client.onEndOfDataBlockEvent(endOfDataBlockEvent);
-            break;
-          } catch (Exception e) {
-            
LOGGER.warn(DataNodeQueryMessages.FAILED_TO_SEND_END_OF_DATA_BLOCK_EVENT, 
attempt, e);
-            if (attempt == MAX_ATTEMPT_TIMES) {
-              
LOGGER.warn(DataNodeQueryMessages.FAILED_TO_SEND_END_OF_DATA_BLOCK_EVENT_2, e);
-              sinkListener.onFailure(SinkChannel.this, e);
-              return;
-            }
-            try {
-              Thread.sleep(retryIntervalInMs);
-            } catch (InterruptedException ex) {
-              Thread.currentThread().interrupt();
-              sinkListener.onFailure(SinkChannel.this, e);
-            }
+        if (!sendNotification(client -> 
client.onEndOfDataBlockEvent(endOfDataBlockEvent), 0)) {
+          return;
+        }
+        synchronized (SinkChannel.this) {
+          if (notificationsStopped) {
+            return;
           }
+          noMoreTsBlocks = true;
         }
-        noMoreTsBlocks = true;
         if (isFinished()) {
           invokeOnFinished();
         }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index 6d33ea28a60..ddca5b2ea20 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -181,6 +181,7 @@ public class FragmentInstanceContext extends QueryContext {
 
   // Outer query deadline (startTime + timeout) for IoTDBLocal UDF
   private long outerQueryDeadlineMs = -1L;
+  private long exchangeDeadlineMs = -1L;
 
   private QueryAttemptRegistry.Registration queryAttemptRegistration;
   private boolean executionInitialized;
@@ -662,6 +663,17 @@ public class FragmentInstanceContext extends QueryContext {
     return outerQueryDeadlineMs;
   }
 
+  void setExchangeDeadlineMs(long coordinatorDeadlineMs) {
+    exchangeDeadlineMs = coordinatorDeadlineMs;
+  }
+
+  public long getExchangeDeadlineMs() {
+    // Legacy coordinators only send the remaining FI timeout. Also honor an 
earlier local FI limit.
+    return exchangeDeadlineMs < 0
+        ? outerQueryDeadlineMs
+        : Math.min(outerQueryDeadlineMs, exchangeDeadlineMs);
+  }
+
   public Optional<Throwable> getFailureCause() {
     return Optional.ofNullable(
         stateMachine.getFailureCauses().stream()
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
index 7926d2b501a..2f7b4d5ff5f 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceManager.java
@@ -180,6 +180,7 @@ public class FragmentInstanceManager {
                                     
DriverTask.computeDeadlineTimeInMs(instance.getTimeOut()),
                                     instance.isDebug(),
                                     instance.isVerbose());
+                            
createdContext.setExchangeDeadlineMs(instance.getQueryDeadlineMs());
                             
createdContext.adoptUDFQueryScope(instance.claimUDFQueryScope());
                             return createdContext;
                           } catch (Throwable t) {
@@ -356,6 +357,7 @@ public class FragmentInstanceManager {
                                   
DriverTask.computeDeadlineTimeInMs(instance.getTimeOut()),
                                   instance.isDebug(),
                                   instance.isVerbose());
+                          
createdContext.setExchangeDeadlineMs(instance.getQueryDeadlineMs());
                           
createdContext.adoptUDFQueryScope(instance.claimUDFQueryScope());
                           return createdContext;
                         } catch (Throwable t) {
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
index 477a48ad56c..b07123bc19c 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/FragmentInstance.java
@@ -65,6 +65,8 @@ public class FragmentInstance implements IConsensusRequest {
   private final TimePredicate globalTimePredicate;
 
   private final long timeOut;
+  // Carried in the optional dispatch envelope. Older coordinators only send 
the FI timeOut.
+  private long queryDeadlineMs = -1;
 
   private boolean isRoot;
 
@@ -364,6 +366,14 @@ public class FragmentInstance implements IConsensusRequest 
{
     return hostDataNode;
   }
 
+  public void setQueryDeadlineMs(long queryDeadlineMs) {
+    this.queryDeadlineMs = queryDeadlineMs;
+  }
+
+  public long getQueryDeadlineMs() {
+    return queryDeadlineMs;
+  }
+
   public long getTimeOut() {
     return timeOut;
   }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
index 35b8c70b328..1c6b05d482a 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
@@ -90,6 +90,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
+import static com.google.common.math.LongMath.saturatedAdd;
 import static com.google.common.util.concurrent.Futures.immediateFuture;
 import static 
org.apache.iotdb.calc.metric.QueryExecutionMetricSet.DISPATCH_READ;
 import static 
org.apache.iotdb.db.utils.ErrorHandlingUtils.onThriftFrameOversizeException;
@@ -508,6 +509,10 @@ public class FragmentInstanceDispatcherImpl implements 
IFragInstanceDispatcher {
   private void dispatchOneInstance(FragmentInstance instance)
       throws FragmentInstanceDispatchException, TException {
     checkNotAborted();
+    if (queryContext.getTimeOut() > 0) {
+      instance.setQueryDeadlineMs(
+          saturatedAdd(queryContext.getStartTime(), 
queryContext.getTimeOut()));
+    }
     TEndPoint endPoint = instance.getHostDataNode().getInternalEndPoint();
     if (isDispatchedToLocal(endPoint)) {
       prepareAdmission(instance, endPoint, null);
@@ -539,6 +544,9 @@ public class FragmentInstanceDispatcherImpl implements 
IFragInstanceDispatcher {
         case READ:
           final TSendFragmentInstanceReq sendFragmentInstanceReq =
               new TSendFragmentInstanceReq(new 
TFragmentInstance(instance.serializeToByteBuffer()));
+          if (instance.getQueryDeadlineMs() >= 0) {
+            
sendFragmentInstanceReq.setQueryDeadlineMs(instance.getQueryDeadlineMs());
+          }
           prepareAdmission(instance, endPoint, client);
           if (instance.hasQueryAttempt()) {
             sendFragmentInstanceReq
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java
index a5573159a94..b400ce92eac 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/MPPDataExchangeManagerTest.java
@@ -20,9 +20,7 @@
 package org.apache.iotdb.db.queryengine.execution.exchange;
 
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.commons.client.ClientPoolFactory;
-import org.apache.iotdb.commons.client.IClientManager;
-import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
 import org.apache.iotdb.commons.memory.MemoryManager;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
@@ -92,7 +90,7 @@ public class MPPDataExchangeManagerTest {
               localMemoryManager,
               new TsBlockSerdeFactory(),
               exchangeExecutor,
-              Mockito.mock(IClientManager.class));
+              Mockito.mock(MPPDataExchangeClientManager.class));
       ISourceHandle sourceHandle =
           exchangeManager.createSourceHandle(
               targetFragmentInstanceId,
@@ -147,9 +145,7 @@ public class MPPDataExchangeManagerTest {
             mockLocalMemoryManager,
             new TsBlockSerdeFactory(),
             Executors.newSingleThreadExecutor(),
-            new IClientManager.Factory<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>()
-                .createClientManager(
-                    new 
ClientPoolFactory.SyncDataNodeMPPDataExchangeServiceClientPoolFactory()));
+            new MPPDataExchangeClientManager());
 
     ISinkHandle shuffleSinkHandle =
         mppDataExchangeManager.createShuffleSinkHandle(
@@ -211,9 +207,7 @@ public class MPPDataExchangeManagerTest {
             mockLocalMemoryManager,
             new TsBlockSerdeFactory(),
             Executors.newSingleThreadExecutor(),
-            new IClientManager.Factory<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>()
-                .createClientManager(
-                    new 
ClientPoolFactory.SyncDataNodeMPPDataExchangeServiceClientPoolFactory()));
+            new MPPDataExchangeClientManager());
 
     ISourceHandle localSourceHandle =
         mppDataExchangeManager.createLocalSourceHandleForFragment(
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelDeadlineTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelDeadlineTest.java
new file mode 100644
index 00000000000..9db213d2a94
--- /dev/null
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelDeadlineTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.execution.exchange;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
+import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
+import org.apache.iotdb.commons.exception.QueryTimeoutException;
+import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
+import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
+import org.apache.iotdb.db.queryengine.common.QueryId;
+import 
org.apache.iotdb.db.queryengine.execution.exchange.MPPDataExchangeManager.SinkListener;
+import 
org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannelIndex;
+import 
org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannelLocation;
+import org.apache.iotdb.db.queryengine.execution.exchange.sink.ISinkHandle;
+import 
org.apache.iotdb.db.queryengine.execution.exchange.sink.ShuffleSinkHandle;
+import org.apache.iotdb.db.queryengine.execution.exchange.sink.SinkChannel;
+import 
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
+import 
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceStateMachine;
+import org.apache.iotdb.db.queryengine.execution.memory.LocalMemoryManager;
+import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstanceId;
+
+import org.apache.thrift.TException;
+import org.junit.After;
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.ArgumentMatchers.isA;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SinkChannelDeadlineTest {
+  private final ExecutorService executor = Executors.newSingleThreadExecutor();
+  private final SinkListener listener = mock(SinkListener.class);
+  private final LocalMemoryManager memory = mock(LocalMemoryManager.class);
+
+  @After
+  public void tearDown() {
+    executor.shutdownNow();
+  }
+
+  private SinkChannel channel(
+      TEndPoint endpoint, long deadline, MPPDataExchangeClientManager clients) 
{
+    
doReturn(Utils.createMockNonBlockedMemoryPool()).when(memory).getQueryPool();
+    return new SinkChannel(
+        endpoint,
+        new TFragmentInstanceId("query", 0, "0"),
+        "exchange",
+        "sink",
+        new TFragmentInstanceId("query", 1, "0"),
+        memory,
+        executor,
+        Utils.createMockTsBlockSerde(1024),
+        listener,
+        false,
+        deadline,
+        clients);
+  }
+
+  private void awaitWorker() throws Exception {
+    executor.submit(() -> {}).get(3, SECONDS);
+  }
+
+  @Test
+  public void queuedEventsExpireWithoutBorrowingAndReleaseMemory() throws 
Exception {
+    MPPDataExchangeClientManager clients = 
mock(MPPDataExchangeClientManager.class);
+    CountDownLatch dequeue = new CountDownLatch(1);
+    executor.submit(
+        () -> {
+          dequeue.await();
+          return null;
+        });
+    long deadline = System.currentTimeMillis() + 200;
+    SinkChannel channel = channel(new TEndPoint("unused", 1), deadline, 
clients);
+    doAnswer(call -> channel.abort())
+        .when(listener)
+        .onFailure(eq(channel), isA(QueryTimeoutException.class));
+    channel.open();
+    channel.send(Utils.createMockTsBlocks(1, 1024).get(0));
+    channel.setNoMoreTsBlocks();
+    assertFalse(dequeue.await(Math.max(0, deadline - 
System.currentTimeMillis()), MILLISECONDS));
+    dequeue.countDown();
+    awaitWorker();
+    verify(clients, never()).borrowClient(any(), anyLong());
+    verify(listener).onFailure(eq(channel), isA(QueryTimeoutException.class));
+    verify(listener, never()).onEndOfBlocks(channel);
+    assertEquals(0, channel.getBufferRetainedSizeInBytes());
+  }
+
+  @Test
+  public void cancelledFragmentSkipsQueuedEndEventBeforeChannelCleanup() 
throws Exception {
+    MPPDataExchangeClientManager clients = 
mock(MPPDataExchangeClientManager.class);
+    CountDownLatch dequeue = new CountDownLatch(1);
+    executor.submit(
+        () -> {
+          dequeue.await();
+          return null;
+        });
+    
doReturn(Utils.createMockNonBlockedMemoryPool()).when(memory).getQueryPool();
+    FragmentInstanceId id =
+        new FragmentInstanceId(new PlanFragmentId(new QueryId("query"), 1), 
"0");
+    FragmentInstanceStateMachine state = new FragmentInstanceStateMachine(id, 
Runnable::run);
+    FragmentInstanceContext context =
+        FragmentInstanceContext.createFragmentInstanceContext(id, state);
+    MPPDataExchangeManager exchange =
+        new MPPDataExchangeManager(memory, new TsBlockSerdeFactory(), 
executor, clients);
+    ISinkHandle sink =
+        exchange.createShuffleSinkHandle(
+            Collections.singletonList(
+                new DownStreamChannelLocation(
+                    new TEndPoint("unused", 1),
+                    new TFragmentInstanceId("query", 0, "0"),
+                    "exchange")),
+            new DownStreamChannelIndex(0),
+            ShuffleSinkHandle.ShuffleStrategyEnum.PLAIN,
+            id.toThrift(),
+            "sink",
+            context);
+    SinkChannel channel = (SinkChannel) sink.getChannel(0);
+    channel.setNoMoreTsBlocks();
+    state.cancel();
+    dequeue.countDown();
+    awaitWorker();
+    verify(clients, never()).borrowClient(any(), anyLong());
+    assertFalse(channel.isNoMoreTsBlocks());
+    assertTrue(state.getFailureCauses().isEmpty());
+    sink.abort();
+  }
+
+  @Test
+  public void retryBackoffSharesTheDeadline() throws Exception {
+    TEndPoint endpoint = new TEndPoint("unused", 1);
+    MPPDataExchangeClientManager clients = 
mock(MPPDataExchangeClientManager.class);
+    SyncDataNodeMPPDataExchangeServiceClient client =
+        mock(SyncDataNodeMPPDataExchangeServiceClient.class);
+    when(clients.borrowClient(eq(endpoint), anyLong())).thenReturn(client);
+    doThrow(new TException("transient 
failure")).when(client).onNewDataBlockEvent(any());
+    SinkChannel channel = channel(endpoint, System.currentTimeMillis() + 800, 
clients);
+    channel.setRetryIntervalInMs(30_000);
+    channel.open();
+    channel.send(Utils.createMockTsBlocks(1, 1024).get(0));
+    awaitWorker();
+    verify(client).onNewDataBlockEvent(any());
+    verify(listener).onFailure(eq(channel), isA(QueryTimeoutException.class));
+    channel.abort();
+  }
+
+  @Test
+  public void abortInterruptsPoolWaitWithoutDiscardingTheHeldConnection() 
throws Exception {
+    MPPDataExchangeClientManager clients = new MPPDataExchangeClientManager();
+    clients.getPool().setMaxTotalPerKey(1);
+    try (ServerSocket server = new ServerSocket(0, 1, 
InetAddress.getLoopbackAddress())) {
+      TEndPoint endpoint =
+          new TEndPoint(server.getInetAddress().getHostAddress(), 
server.getLocalPort());
+      try (SyncDataNodeMPPDataExchangeServiceClient held = 
clients.borrowClient(endpoint);
+          Socket peer = server.accept()) {
+        SinkChannel channel = channel(endpoint, System.currentTimeMillis() + 
20_000, clients);
+        channel.setNoMoreTsBlocks();
+        long start = System.nanoTime();
+        while (clients.getPool().getNumWaiters() == 0
+            && System.nanoTime() - start < SECONDS.toNanos(3)) {
+          Thread.sleep(10);
+        }
+        assertEquals(1, clients.getPool().getNumWaiters());
+        channel.abort();
+        awaitWorker();
+        verify(listener, never()).onFailure(any(), any());
+        assertEquals(1, clients.getPool().getNumActive());
+      }
+      assertEquals(1, clients.getPool().getNumIdle());
+    } finally {
+      clients.close();
+    }
+  }
+
+  @Test
+  public void silentPeerReleasesWorkerAtDeadline() throws Exception {
+    verifyBlockedRead(false);
+  }
+
+  @Test
+  public void abortInterruptsReadAndLeavesOtherConnectionUsable() throws 
Exception {
+    verifyBlockedRead(true);
+  }
+
+  private void verifyBlockedRead(boolean abort) throws Exception {
+    MPPDataExchangeClientManager clients = new MPPDataExchangeClientManager();
+    try (ServerSocket server = new ServerSocket(0, 2, 
InetAddress.getLoopbackAddress())) {
+      server.setSoTimeout(3_000);
+      TEndPoint endpoint =
+          new TEndPoint(server.getInetAddress().getHostAddress(), 
server.getLocalPort());
+      // Warm up the connection and keep another idle connection to the same 
endpoint.
+      try (SyncDataNodeMPPDataExchangeServiceClient first = 
clients.borrowClient(endpoint);
+          SyncDataNodeMPPDataExchangeServiceClient second = 
clients.borrowClient(endpoint)) {}
+      long borrowed = clients.getPool().getBorrowedCount();
+      // The pool is LIFO, and the first client was returned last.
+      try (Socket peer = server.accept();
+          Socket other = server.accept()) {
+        peer.setSoTimeout(3_000);
+        long deadline = System.currentTimeMillis() + (abort ? 20_000 : 800);
+        SinkChannel channel = channel(endpoint, deadline, clients);
+        channel.setNoMoreTsBlocks();
+        assertTrue(peer.getInputStream().read() >= 0);
+        if (abort) {
+          channel.abort();
+        }
+        awaitWorker();
+        assertEquals(borrowed + 1, clients.getPool().getBorrowedCount());
+        assertEquals(0, clients.getPool().getNumActive());
+        assertEquals(1, clients.getPool().getNumIdle());
+        verify(listener, never()).onEndOfBlocks(channel);
+        if (abort) {
+          verify(listener, never()).onFailure(any(), any());
+        } else {
+          verify(listener).onFailure(eq(channel), 
isA(QueryTimeoutException.class));
+        }
+      }
+    } finally {
+      clients.close();
+    }
+  }
+}
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java
index 7030b5c210c..12e2cc105fe 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelFailurePropagationTest.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.queryengine.execution.exchange;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
 import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
 import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
 import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
 import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
@@ -79,7 +80,7 @@ public class SinkChannelFailurePropagationTest {
               Mockito.mock(LocalMemoryManager.class),
               new TsBlockSerdeFactory(),
               exchangeExecutor,
-              Mockito.mock(IClientManager.class));
+              Mockito.mock(MPPDataExchangeClientManager.class));
       AtomicInteger remainingChannels = new AtomicInteger(1);
       AtomicBoolean hasChannelFailedOrAborted = new AtomicBoolean(false);
       AtomicBoolean callbackInvoked = new AtomicBoolean(false);
@@ -131,8 +132,8 @@ public class SinkChannelFailurePropagationTest {
       MemoryPool memoryPool = Utils.createMockNonBlockedMemoryPool();
       Mockito.when(localMemoryManager.getQueryPool()).thenReturn(memoryPool);
 
-      IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
exchangeClientManager =
-          Mockito.mock(IClientManager.class);
+      MPPDataExchangeClientManager exchangeClientManager =
+          Mockito.mock(MPPDataExchangeClientManager.class);
       SyncDataNodeMPPDataExchangeServiceClient exchangeClient =
           Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class);
       TException expectedFailure =
@@ -143,7 +144,8 @@ public class SinkChannelFailurePropagationTest {
                   new TApplicationException(
                       TApplicationException.INTERNAL_ERROR,
                       "Internal error processing onNewDataBlockEvent")));
-      
Mockito.when(exchangeClientManager.borrowClient(secondEndpoint)).thenReturn(exchangeClient);
+      Mockito.when(exchangeClientManager.borrowClient(secondEndpoint, 
Long.MAX_VALUE))
+          .thenReturn(exchangeClient);
       Mockito.doThrow(expectedFailure)
           .when(exchangeClient)
           .onNewDataBlockEvent(Mockito.any(TNewDataBlockEvent.class));
@@ -226,12 +228,13 @@ public class SinkChannelFailurePropagationTest {
       MemoryPool memoryPool = Utils.createMockNonBlockedMemoryPool();
       Mockito.when(localMemoryManager.getQueryPool()).thenReturn(memoryPool);
 
-      IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
exchangeClientManager =
-          Mockito.mock(IClientManager.class);
+      MPPDataExchangeClientManager exchangeClientManager =
+          Mockito.mock(MPPDataExchangeClientManager.class);
       SyncDataNodeMPPDataExchangeServiceClient exchangeClient =
           Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class);
       TException expectedFailure = new TException("injected 
onNewDataBlockEvent failure");
-      
Mockito.when(exchangeClientManager.borrowClient(exchangeEndpoint)).thenReturn(exchangeClient);
+      Mockito.when(exchangeClientManager.borrowClient(exchangeEndpoint, 
Long.MAX_VALUE))
+          .thenReturn(exchangeClient);
       Mockito.doThrow(expectedFailure)
           .when(exchangeClient)
           .onNewDataBlockEvent(Mockito.any(TNewDataBlockEvent.class));
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java
index f22725c0fea..220993bbfad 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/exchange/SinkChannelTest.java
@@ -20,8 +20,8 @@
 package org.apache.iotdb.db.queryengine.execution.exchange;
 
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.commons.client.IClientManager;
 import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.client.sync.MPPDataExchangeClientManager;
 import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
 import org.apache.iotdb.commons.memory.MemoryManager;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -67,13 +67,14 @@ public class SinkChannelTest {
     LocalMemoryManager mockLocalMemoryManager = 
Mockito.mock(LocalMemoryManager.class);
     MemoryPool mockMemoryPool = Utils.createMockNonBlockedMemoryPool();
     
Mockito.when(mockLocalMemoryManager.getQueryPool()).thenReturn(mockMemoryPool);
-    IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
mockClientManager =
-        Mockito.mock(IClientManager.class);
+    MPPDataExchangeClientManager mockClientManager =
+        Mockito.mock(MPPDataExchangeClientManager.class);
     // Construct a mock client.
     SyncDataNodeMPPDataExchangeServiceClient mockClient =
         Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class);
     try {
-      
Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient);
+      Mockito.when(mockClientManager.borrowClient(remoteEndpoint, 
Long.MAX_VALUE))
+          .thenReturn(mockClient);
       Mockito.doNothing()
           .when(mockClient)
           .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class));
@@ -232,13 +233,14 @@ public class SinkChannelTest {
     SinkListener mockSinkListener = Mockito.mock(SinkListener.class);
     // Construct several mock TsBlock(s).
     List<TsBlock> mockTsBlocks = Utils.createMockTsBlocks(numOfMockTsBlock, 
mockTsBlockSize);
-    IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
mockClientManager =
-        Mockito.mock(IClientManager.class);
+    MPPDataExchangeClientManager mockClientManager =
+        Mockito.mock(MPPDataExchangeClientManager.class);
     // Construct a mock client.
     SyncDataNodeMPPDataExchangeServiceClient mockClient =
         Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class);
     try {
-      
Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient);
+      Mockito.when(mockClientManager.borrowClient(remoteEndpoint, 
Long.MAX_VALUE))
+          .thenReturn(mockClient);
       Mockito.doNothing()
           .when(mockClient)
           .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class));
@@ -436,14 +438,15 @@ public class SinkChannelTest {
     SinkListener mockSinkListener = Mockito.mock(SinkListener.class);
     // Construct several mock TsBlock(s).
     List<TsBlock> mockTsBlocks = Utils.createMockTsBlocks(numOfMockTsBlock, 
mockTsBlockSize);
-    IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
mockClientManager =
-        Mockito.mock(IClientManager.class);
+    MPPDataExchangeClientManager mockClientManager =
+        Mockito.mock(MPPDataExchangeClientManager.class);
     // Construct a mock client.
     SyncDataNodeMPPDataExchangeServiceClient mockClient =
         Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class);
     TException mockException = new TException("Mock exception");
     try {
-      
Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient);
+      Mockito.when(mockClientManager.borrowClient(remoteEndpoint, 
Long.MAX_VALUE))
+          .thenReturn(mockClient);
       Mockito.doThrow(mockException)
           .when(mockClient)
           .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class));
@@ -547,13 +550,14 @@ public class SinkChannelTest {
     SinkListener mockSinkListener = Mockito.mock(SinkListener.class);
     // Construct several mock TsBlock(s).
     List<TsBlock> mockTsBlocks = Utils.createMockTsBlocks(numOfMockTsBlock, 
mockTsBlockSize);
-    IClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
mockClientManager =
-        Mockito.mock(IClientManager.class);
+    MPPDataExchangeClientManager mockClientManager =
+        Mockito.mock(MPPDataExchangeClientManager.class);
     // Construct a mock client.
     SyncDataNodeMPPDataExchangeServiceClient mockClient =
         Mockito.mock(SyncDataNodeMPPDataExchangeServiceClient.class);
     try {
-      
Mockito.when(mockClientManager.borrowClient(remoteEndpoint)).thenReturn(mockClient);
+      Mockito.when(mockClientManager.borrowClient(remoteEndpoint, 
Long.MAX_VALUE))
+          .thenReturn(mockClient);
       Mockito.doNothing()
           .when(mockClient)
           .onEndOfDataBlockEvent(Mockito.any(TEndOfDataBlockEvent.class));
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java
index 4c103afff0d..304b5592eda 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContextTest.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.commons.service.metric.enums.Metric;
 import org.apache.iotdb.commons.service.metric.enums.Tag;
 import org.apache.iotdb.db.queryengine.common.FragmentInstanceId;
 import org.apache.iotdb.db.queryengine.common.PlanFragmentId;
+import org.apache.iotdb.db.queryengine.common.QueryId;
 import org.apache.iotdb.db.queryengine.execution.aggregation.TreeAggregator;
 import 
org.apache.iotdb.db.queryengine.execution.aggregation.timerangeiterator.ITimeRangeIterator;
 import org.apache.iotdb.db.queryengine.execution.driver.DriverContext;
@@ -75,6 +76,23 @@ import static org.junit.Assert.assertTrue;
 
 public class FragmentInstanceContextTest {
 
+  @Test
+  public void exchangeDeadlineHonorsBothCoordinatorAndLegacyFragmentLimits() {
+    FragmentInstanceId id =
+        new FragmentInstanceId(new PlanFragmentId(new QueryId("deadline"), 0), 
"0");
+    FragmentInstanceStateMachine state = new FragmentInstanceStateMachine(id, 
Runnable::run);
+    long localDeadline = System.currentTimeMillis() + 30_000;
+    FragmentInstanceContext context =
+        FragmentInstanceContext.createFragmentInstanceContext(
+            id, state, null, localDeadline, false, false);
+    assertEquals(localDeadline, context.getExchangeDeadlineMs());
+    context.setExchangeDeadlineMs(localDeadline - 1000);
+    assertEquals(localDeadline - 1000, context.getExchangeDeadlineMs());
+    assertEquals(localDeadline, context.getOuterQueryDeadlineMs());
+    context.setExchangeDeadlineMs(Long.MAX_VALUE);
+    assertEquals(localDeadline, context.getExchangeDeadlineMs());
+  }
+
   @Test
   public void testConcurrentQueryDataSourceStatistics() throws 
InterruptedException {
     ExecutorService instanceNotificationExecutor =
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java
index 5846254faad..aeaad8b934b 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherLifecycleTest.java
@@ -119,6 +119,8 @@ public class FragmentInstanceDispatcherLifecycleTest {
   public void transportRetryReusesTokenAndStopFreezesIssuedTokens() throws 
Exception {
     FragmentInstanceDispatcherImpl dispatcher = dispatcher();
     context.prepareForRetry();
+    context.setTimeOut(30_000);
+    context.setStartTime(System.currentTimeMillis() - 3_200);
     List<TSendFragmentInstanceReq> requests = new ArrayList<>();
     when(client.sendFragmentInstance(any()))
         .thenAnswer(
@@ -133,6 +135,8 @@ public class FragmentInstanceDispatcherLifecycleTest {
     FragmentInstance instance = instance(0);
     assertTrue(dispatcher.dispatch(null, 
Collections.singletonList(instance)).get().isSuccessful());
     assertEquals(2, requests.size());
+    assertEquals(context.getStartTime() + 30_000, 
requests.get(0).getQueryDeadlineMs());
+    assertEquals(requests.get(0).getQueryDeadlineMs(), 
requests.get(1).getQueryDeadlineMs());
     assertEquals(0, requests.get(0).getQueryAttemptId());
     assertEquals(token, requests.get(0).getAdmissionToken());
     assertEquals(requests.get(0).getAdmissionToken(), 
requests.get(1).getAdmissionToken());
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java
index d77cdc8e1a7..7cde933e89d 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/scheduler/QueryAttemptProtocolTest.java
@@ -49,6 +49,7 @@ public class QueryAttemptProtocolTest {
     assertEquals(original, received);
     assertFalse(received.isSetQueryAttemptId());
     assertFalse(received.isSetAdmissionToken());
+    assertFalse(received.isSetQueryDeadlineMs());
     TCancelQueryReq cancel = new TCancelQueryReq("query", 
Collections.emptyList(), true);
     TCancelQueryReq receivedCancel = new TCancelQueryReq();
     new TDeserializer().deserialize(receivedCancel, new 
TSerializer().serialize(cancel));
@@ -62,6 +63,7 @@ public class QueryAttemptProtocolTest {
     TFragmentInstance body = new TFragmentInstance(ByteBuffer.wrap(new byte[] 
{1, 2, 3}));
     TSendFragmentInstanceReq request =
         new TSendFragmentInstanceReq(body)
+            .setQueryDeadlineMs(123456789L)
             .setQueryAttemptId(2)
             .setAdmissionToken(new TQueryAdmissionToken("receiver", 123450L, 
1000L));
     byte[] serialized = new TSerializer().serialize(request);
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java
index 336658fd5dd..b320b43393a 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientManager.java
@@ -38,7 +38,7 @@ public class ClientManager<K, V> implements IClientManager<K, 
V> {
 
   private final GenericKeyedObjectPool<K, V> pool;
 
-  ClientManager(IClientPoolFactory<K, V> factory) {
+  protected ClientManager(IClientPoolFactory<K, V> factory) {
     pool = factory.createClientPool(this);
   }
 
@@ -49,11 +49,22 @@ public class ClientManager<K, V> implements 
IClientManager<K, V> {
 
   @Override
   public V borrowClient(K node) throws ClientManagerException {
+    return borrowClientWithTimeout(node, Long.MAX_VALUE);
+  }
+
+  protected V borrowClientWithTimeout(K node, long maxWaitMs) throws 
ClientManagerException {
     if (node == null) {
       throw new BorrowNullClientManagerException();
     }
     try {
-      return pool.borrowObject(node);
+      if (maxWaitMs == Long.MAX_VALUE) {
+        return pool.borrowObject(node);
+      }
+      long poolWaitMs = pool.getMaxWaitDuration().toMillis();
+      return pool.borrowObject(node, poolWaitMs < 0 ? maxWaitMs : 
Math.min(poolWaitMs, maxWaitMs));
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new ClientManagerException(e);
     } catch (Exception e) {
       throw new ClientManagerException(e);
     }
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java
index ce5c8366889..a3b8e9af7e7 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/ClientPoolFactory.java
@@ -34,7 +34,6 @@ import 
org.apache.iotdb.commons.client.property.ThriftClientProperty.DefaultProp
 import org.apache.iotdb.commons.client.sync.SyncAINodeClient;
 import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
 import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
-import 
org.apache.iotdb.commons.client.sync.SyncDataNodeMPPDataExchangeServiceClient;
 import org.apache.iotdb.commons.client.sync.SyncIoTConsensusV2ServiceClient;
 import org.apache.iotdb.commons.concurrent.ThreadName;
 import org.apache.iotdb.commons.conf.CommonConfig;
@@ -251,30 +250,6 @@ public class ClientPoolFactory {
     }
   }
 
-  public static class SyncDataNodeMPPDataExchangeServiceClientPoolFactory
-      implements IClientPoolFactory<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient> {
-
-    @Override
-    public GenericKeyedObjectPool<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>
-        createClientPool(
-            ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
manager) {
-      GenericKeyedObjectPool<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient> clientPool =
-          new GenericKeyedObjectPool<>(
-              new SyncDataNodeMPPDataExchangeServiceClient.Factory(
-                  manager,
-                  new ThriftClientProperty.Builder()
-                      
.setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS())
-                      
.setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled())
-                      .build()),
-              new 
ClientPoolProperty.Builder<SyncDataNodeMPPDataExchangeServiceClient>()
-                  .build()
-                  .getConfig());
-      ClientManagerMetrics.getInstance()
-          .registerClientManager(this.getClass().getSimpleName(), clientPool);
-      return clientPool;
-    }
-  }
-
   public static class AsyncDataNodeMPPDataExchangeServiceClientPoolFactory
       implements IClientPoolFactory<TEndPoint, 
AsyncDataNodeMPPDataExchangeServiceClient> {
 
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManager.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManager.java
new file mode 100644
index 00000000000..58bbf7494d3
--- /dev/null
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManager.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.client.sync;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.ClientManager;
+import org.apache.iotdb.commons.client.ClientManagerMetrics;
+import org.apache.iotdb.commons.client.IClientPoolFactory;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.client.property.ClientPoolProperty;
+import org.apache.iotdb.commons.client.property.ThriftClientProperty;
+import org.apache.iotdb.commons.concurrent.IoTDBDaemonThreadFactory;
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.exception.QueryTimeoutException;
+
+import org.apache.commons.pool2.impl.GenericKeyedObjectPool;
+import org.apache.thrift.transport.TTransportException;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+public class MPPDataExchangeClientManager
+    extends ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
{
+
+  // Native name resolution and TLS store loading can ignore interrupts. Keep 
these cold-path
+  // operations off exchange workers. Each waiting borrower removes its task 
on cancellation.
+  private final ThreadPoolExecutor connectionSetup =
+      new ThreadPoolExecutor(
+          4,
+          4,
+          1,
+          TimeUnit.MINUTES,
+          new LinkedBlockingQueue<>(),
+          new 
IoTDBDaemonThreadFactory(ThreadName.MPP_DATA_EXCHANGE_CONNECTION_SETUP.getName()));
+
+  public MPPDataExchangeClientManager() {
+    super(new SyncDataNodeMPPDataExchangeServiceClientPoolFactory());
+    connectionSetup.allowCoreThreadTimeOut(true);
+  }
+
+  @Override
+  public SyncDataNodeMPPDataExchangeServiceClient borrowClient(TEndPoint 
endpoint)
+      throws ClientManagerException {
+    return borrowClient(endpoint, Long.MAX_VALUE);
+  }
+
+  public SyncDataNodeMPPDataExchangeServiceClient borrowClient(TEndPoint 
endpoint, long deadlineMs)
+      throws ClientManagerException {
+    SyncDataNodeMPPDataExchangeServiceClient client =
+        borrowClientWithTimeout(endpoint, remainingMillis(deadlineMs));
+    boolean ready = false;
+    try {
+      client.prepare(deadlineMs);
+      ready = true;
+      return client;
+    } catch (Exception e) {
+      throw new ClientManagerException(e);
+    } finally {
+      if (!ready) {
+        try {
+          client.invalidate();
+        } finally {
+          client.close();
+        }
+      }
+    }
+  }
+
+  <T> T prepareConnection(Callable<T> prepare, int timeoutMs) throws 
TTransportException {
+    FutureTask<T> task = new FutureTask<>(prepare);
+    connectionSetup.execute(task);
+    try {
+      return timeoutMs == 0 ? task.get() : task.get(timeoutMs, 
TimeUnit.MILLISECONDS);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      throw new TTransportException(e);
+    } catch (TimeoutException e) {
+      throw new TTransportException(TTransportException.TIMED_OUT, e);
+    } catch (ExecutionException e) {
+      throw new TTransportException(e.getCause());
+    } finally {
+      if (task.cancel(true)) {
+        connectionSetup.remove(task);
+      }
+    }
+  }
+
+  @Override
+  public void close() {
+    for (Runnable queued : connectionSetup.shutdownNow()) {
+      ((FutureTask<?>) queued).cancel(false);
+    }
+    super.close();
+  }
+
+  public static long remainingMillis(long deadlineMs) {
+    if (deadlineMs == Long.MAX_VALUE) {
+      return Long.MAX_VALUE;
+    }
+    long remaining = deadlineMs - System.currentTimeMillis();
+    if (remaining <= 0) {
+      throw new QueryTimeoutException();
+    }
+    return remaining;
+  }
+
+  private static class SyncDataNodeMPPDataExchangeServiceClientPoolFactory
+      implements IClientPoolFactory<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient> {
+
+    @Override
+    public GenericKeyedObjectPool<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient>
+        createClientPool(
+            ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
manager) {
+      CommonConfig conf = CommonDescriptor.getInstance().getConfig();
+      GenericKeyedObjectPool<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient> clientPool =
+          new GenericKeyedObjectPool<>(
+              new SyncDataNodeMPPDataExchangeServiceClient.Factory(
+                  manager,
+                  new ThriftClientProperty.Builder()
+                      
.setConnectionTimeoutMs(conf.getDnConnectionTimeoutInMS())
+                      
.setRpcThriftCompressionEnabled(conf.isRpcThriftCompressionEnabled())
+                      .build()),
+              new 
ClientPoolProperty.Builder<SyncDataNodeMPPDataExchangeServiceClient>()
+                  .build()
+                  .getConfig());
+      ClientManagerMetrics.getInstance()
+          .registerClientManager(this.getClass().getSimpleName(), clientPool);
+      return clientPool;
+    }
+  }
+}
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java
index b4393bfd4ee..48beeab7c92 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/sync/SyncDataNodeMPPDataExchangeServiceClient.java
@@ -26,57 +26,128 @@ import 
org.apache.iotdb.commons.client.factory.ThriftClientFactory;
 import org.apache.iotdb.commons.client.property.ThriftClientProperty;
 import org.apache.iotdb.commons.conf.CommonConfig;
 import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.i18n.ServiceMessages;
 import org.apache.iotdb.mpp.rpc.thrift.MPPDataExchangeService;
 import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory;
-import org.apache.iotdb.rpc.TConfigurationConst;
+import org.apache.iotdb.rpc.RpcSslUtils;
 import org.apache.iotdb.rpc.TimeoutChangeableTransport;
 
+import com.google.common.net.InetAddresses;
 import org.apache.commons.pool2.PooledObject;
 import org.apache.commons.pool2.impl.DefaultPooledObject;
 import org.apache.thrift.transport.TSocket;
+import org.apache.thrift.transport.TTransport;
 import org.apache.thrift.transport.TTransportException;
+import org.apache.tsfile.external.commons.lang3.StringUtils;
 
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
 import java.net.SocketException;
+import java.nio.channels.SocketChannel;
+import java.security.GeneralSecurityException;
 
 public class SyncDataNodeMPPDataExchangeServiceClient extends 
MPPDataExchangeService.Client
     implements ThriftClient, AutoCloseable {
 
-  private final boolean printLogWhenEncounterException;
-  private final TEndPoint endpoint;
-  private final ClientManager<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient> clientManager;
   private static final CommonConfig commonConfig = 
CommonDescriptor.getInstance().getConfig();
 
-  public SyncDataNodeMPPDataExchangeServiceClient(
+  private final ThriftClientProperty property;
+  private final TEndPoint endpoint;
+  private final MPPDataExchangeClientManager clientManager;
+  private boolean invalidated;
+  private boolean restoreTimeout;
+
+  // The pool allocates the client before connecting so a stalled connect 
cannot block other
+  // borrowers inside Commons Pool's per-key makeObjectCountLock.
+  protected SyncDataNodeMPPDataExchangeServiceClient(
       ThriftClientProperty property,
       TEndPoint endpoint,
-      ClientManager<TEndPoint, SyncDataNodeMPPDataExchangeServiceClient> 
clientManager)
-      throws TTransportException {
-    super(
-        property
-            .getProtocolFactory()
-            .getProtocol(
-                commonConfig.isEnableInternalSSL()
-                    ? DeepCopyRpcTransportFactory.INSTANCE.getTransport(
-                        endpoint.getIp(),
-                        endpoint.getPort(),
-                        property.getConnectionTimeoutMs(),
-                        commonConfig.getTrustStorePath(),
-                        commonConfig.getTrustStorePwd(),
-                        commonConfig.getKeyStorePath(),
-                        commonConfig.getKeyStorePwd(),
-                        commonConfig.getSslProtocol())
-                    : DeepCopyRpcTransportFactory.INSTANCE.getTransport(
-                        new TSocket(
-                            TConfigurationConst.defaultTConfiguration,
-                            endpoint.getIp(),
-                            endpoint.getPort(),
-                            property.getConnectionTimeoutMs()))));
-    this.printLogWhenEncounterException = 
property.isPrintLogWhenEncounterException();
+      MPPDataExchangeClientManager clientManager) {
+    super(null);
+    this.property = property;
     this.endpoint = endpoint;
     this.clientManager = clientManager;
-    if (!getInputProtocol().getTransport().isOpen()) {
-      getInputProtocol().getTransport().open();
+  }
+
+  final void prepare(long deadlineMs) throws TTransportException, IOException {
+    if (iprot_ == null) {
+      // A channel-backed socket makes blocking connects, writes and reads 
interruptible,
+      // including reads during the TLS handshake.
+      TTransport transport = connect(SocketChannel.open().socket(), 
deadlineMs);
+      try {
+        iprot_ = oprot_ = property.getProtocolFactory().getProtocol(transport);
+      } catch (RuntimeException | Error e) {
+        transport.close();
+        throw e;
+      }
+    }
+    if (deadlineMs != Long.MAX_VALUE) {
+      setTimeout(rpcTimeout(deadlineMs));
+    }
+  }
+
+  private record Peer(InetSocketAddress address, SSLSocketFactory sslFactory) 
{}
+
+  private Peer resolvePeer(long deadlineMs) throws TTransportException {
+    String host = endpoint.getIp();
+    int port = endpoint.getPort();
+    boolean ssl = commonConfig.isEnableInternalSSL();
+    if (!ssl && InetAddresses.isInetAddress(host)) {
+      return new Peer(new InetSocketAddress(InetAddresses.forString(host), 
port), null);
+    }
+    return clientManager.prepareConnection(
+        () -> new Peer(new InetSocketAddress(host, port), ssl ? 
sslSocketFactory() : null),
+        rpcTimeout(deadlineMs));
+  }
+
+  final TTransport connect(Socket socket, long deadlineMs) throws IOException, 
TTransportException {
+    try {
+      Peer peer = resolvePeer(deadlineMs);
+      socket.connect(peer.address(), rpcTimeout(deadlineMs));
+      if (peer.sslFactory() != null) {
+        SSLSocket ssl =
+            (SSLSocket)
+                peer.sslFactory().createSocket(socket, endpoint.getIp(), 
endpoint.getPort(), true);
+        socket = ssl;
+        SSLParameters params = ssl.getSSLParameters();
+        params.setEndpointIdentificationAlgorithm("HTTPS");
+        ssl.setSSLParameters(params);
+      }
+      socket.setSoTimeout(rpcTimeout(deadlineMs));
+      return DeepCopyRpcTransportFactory.INSTANCE.getTransport(new 
TSocket(socket));
+    } catch (IOException | TTransportException | RuntimeException | Error e) {
+      socket.close();
+      throw e;
+    }
+  }
+
+  private static SSLSocketFactory sslSocketFactory()
+      throws GeneralSecurityException, IOException, TTransportException {
+    if (StringUtils.isBlank(commonConfig.getKeyStorePath())
+        && StringUtils.isBlank(commonConfig.getTrustStorePath())) {
+      throw new 
TTransportException(ServiceMessages.KEYSTORE_OR_TRUSTSTORE_NOT_FOUND);
+    }
+    return RpcSslUtils.createSSLContext(
+            commonConfig.getKeyStorePath(),
+            commonConfig.getKeyStorePwd(),
+            commonConfig.getTrustStorePath(),
+            commonConfig.getTrustStorePwd(),
+            commonConfig.getSslProtocol())
+        .getSocketFactory();
+  }
+
+  private int rpcTimeout(long deadlineMs) {
+    int configured = property.getConnectionTimeoutMs();
+    if (deadlineMs == Long.MAX_VALUE) {
+      return configured;
     }
+    long remaining = MPPDataExchangeClientManager.remainingMillis(deadlineMs);
+    return (int) Math.min(remaining, configured > 0 ? configured : 
Integer.MAX_VALUE);
   }
 
   public int getTimeout() throws SocketException {
@@ -84,6 +155,7 @@ public class SyncDataNodeMPPDataExchangeServiceClient 
extends MPPDataExchangeSer
   }
 
   public void setTimeout(int timeout) {
+    restoreTimeout = timeout != property.getConnectionTimeoutMs();
     // the same transport is used in both input and output
     ((TimeoutChangeableTransport) 
(getInputProtocol().getTransport())).setTimeout(timeout);
   }
@@ -95,17 +167,25 @@ public class SyncDataNodeMPPDataExchangeServiceClient 
extends MPPDataExchangeSer
 
   @Override
   public void invalidate() {
-    getInputProtocol().getTransport().close();
+    if (!invalidated) {
+      invalidated = true;
+      if (iprot_ != null) {
+        iprot_.getTransport().close();
+      }
+    }
   }
 
   @Override
   public void invalidateAll() {
-    clientManager.clear(endpoint);
+    // An interrupt closes just this channel. Other clients to the same peer 
remain usable.
+    if (!Thread.currentThread().isInterrupted()) {
+      clientManager.clear(endpoint);
+    }
   }
 
   @Override
   public boolean printLogWhenEncounterException() {
-    return printLogWhenEncounterException;
+    return property.isPrintLogWhenEncounterException();
   }
 
   @Override
@@ -113,7 +193,7 @@ public class SyncDataNodeMPPDataExchangeServiceClient 
extends MPPDataExchangeSer
     return String.format("SyncDataNodeMPPDataExchangeServiceClient{%s}", 
endpoint);
   }
 
-  public static class Factory
+  static class Factory
       extends ThriftClientFactory<TEndPoint, 
SyncDataNodeMPPDataExchangeServiceClient> {
 
     public Factory(
@@ -134,17 +214,28 @@ public class SyncDataNodeMPPDataExchangeServiceClient 
extends MPPDataExchangeSer
       return new DefaultPooledObject<>(
           SyncThriftClientWithErrorHandler.newErrorHandler(
               SyncDataNodeMPPDataExchangeServiceClient.class,
-              SyncDataNodeMPPDataExchangeServiceClient.class.getConstructor(
-                  thriftClientProperty.getClass(), endpoint.getClass(), 
clientManager.getClass()),
+              
SyncDataNodeMPPDataExchangeServiceClient.class.getDeclaredConstructor(
+                  ThriftClientProperty.class, TEndPoint.class, 
MPPDataExchangeClientManager.class),
               thriftClientProperty,
               endpoint,
               clientManager));
     }
 
+    @Override
+    public void passivateObject(
+        TEndPoint endpoint, 
PooledObject<SyncDataNodeMPPDataExchangeServiceClient> pooledObject) {
+      SyncDataNodeMPPDataExchangeServiceClient client = 
pooledObject.getObject();
+      if (client.restoreTimeout) {
+        client.setTimeout(thriftClientProperty.getConnectionTimeoutMs());
+      }
+    }
+
     @Override
     public boolean validateObject(
         TEndPoint endpoint, 
PooledObject<SyncDataNodeMPPDataExchangeServiceClient> pooledObject) {
-      return 
pooledObject.getObject().getInputProtocol().getTransport().isOpen();
+      SyncDataNodeMPPDataExchangeServiceClient client = 
pooledObject.getObject();
+      return !client.invalidated
+          && (client.iprot_ == null || client.iprot_.getTransport().isOpen());
     }
   }
 }
diff --git 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
index f0d3b02dcee..cc27a56171f 100644
--- 
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
+++ 
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/concurrent/ThreadName.java
@@ -41,6 +41,7 @@ public enum ThreadName {
   // -------------------------- MPP --------------------------
   MPP_COORDINATOR_SCHEDULED_EXECUTOR("MPP-Coordinator-Scheduled-Executor"),
   MPP_DATA_EXCHANGE_TASK_EXECUTOR("MPP-Data-Exchange-Task-Executors"),
+  MPP_DATA_EXCHANGE_CONNECTION_SETUP("MPP-Data-Exchange-Connection-Setup"),
   ASYNC_DATANODE_CLIENT_POOL("AsyncDataNodeInternalServiceClientPool"),
   MPP_DATA_EXCHANGE_RPC_SERVICE("MPPDataExchangeRPC-Service"),
   MPP_DATA_EXCHANGE_RPC_PROCESSOR("MPPDataExchangeRPC-Processor"),
@@ -232,6 +233,7 @@ public enum ThreadName {
           Arrays.asList(
               MPP_COORDINATOR_SCHEDULED_EXECUTOR,
               MPP_DATA_EXCHANGE_TASK_EXECUTOR,
+              MPP_DATA_EXCHANGE_CONNECTION_SETUP,
               ASYNC_DATANODE_CLIENT_POOL,
               MPP_DATA_EXCHANGE_RPC_SERVICE,
               MPP_DATA_EXCHANGE_RPC_PROCESSOR,
diff --git 
a/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManagerTest.java
 
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManagerTest.java
new file mode 100644
index 00000000000..dbd1b76352f
--- /dev/null
+++ 
b/iotdb-core/node-commons/src/test/java/org/apache/iotdb/commons/client/sync/MPPDataExchangeClientManagerTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.client.sync;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.client.property.ThriftClientProperty;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.exception.QueryTimeoutException;
+
+import org.apache.thrift.transport.TTransportException;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketAddress;
+import java.net.SocketTimeoutException;
+import java.time.Duration;
+import java.util.concurrent.CountDownLatch;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+public class MPPDataExchangeClientManagerTest {
+  @Test
+  public void reusedConnectionUsesEachBorrowersTimeout() throws Exception {
+    MPPDataExchangeClientManager manager = new MPPDataExchangeClientManager();
+    int configured = 
CommonDescriptor.getInstance().getConfig().getDnConnectionTimeoutInMS();
+    try (ServerSocket server = new ServerSocket(0, 1, 
InetAddress.getLoopbackAddress())) {
+      TEndPoint endpoint =
+          new TEndPoint(server.getInetAddress().getHostAddress(), 
server.getLocalPort());
+      SyncDataNodeMPPDataExchangeServiceClient first;
+      try (SyncDataNodeMPPDataExchangeServiceClient client = 
manager.borrowClient(endpoint)) {
+        first = client;
+        assertEquals(configured, client.getTimeout());
+      }
+      try (Socket peer = server.accept()) {
+        try (SyncDataNodeMPPDataExchangeServiceClient shortQuery =
+            manager.borrowClient(endpoint, System.currentTimeMillis() + 800)) {
+          assertSame(first, shortQuery);
+          assertTrue(shortQuery.getTimeout() > 0 && shortQuery.getTimeout() <= 
800);
+        }
+        try (SyncDataNodeMPPDataExchangeServiceClient legacy = 
manager.borrowClient(endpoint)) {
+          assertSame(first, legacy);
+          assertEquals(configured, legacy.getTimeout());
+        }
+        try (SyncDataNodeMPPDataExchangeServiceClient longQuery =
+            manager.borrowClient(endpoint, System.currentTimeMillis() + 
20_000)) {
+          assertSame(first, longQuery);
+          assertTrue(longQuery.getTimeout() > 800 && longQuery.getTimeout() <= 
20_000);
+        }
+        try (SyncDataNodeMPPDataExchangeServiceClient capped =
+            manager.borrowClient(endpoint, System.currentTimeMillis() + 2L * 
configured)) {
+          assertEquals(configured, capped.getTimeout());
+        }
+      }
+    } finally {
+      manager.close();
+    }
+  }
+
+  @Test(timeout = 10_000)
+  public void exhaustedPoolHonorsBothQueryAndPoolWaitLimits() throws Exception 
{
+    MPPDataExchangeClientManager manager = new MPPDataExchangeClientManager();
+    manager.getPool().setMaxTotalPerKey(1);
+    try (ServerSocket server = new ServerSocket(0, 1, 
InetAddress.getLoopbackAddress())) {
+      TEndPoint endpoint =
+          new TEndPoint(server.getInetAddress().getHostAddress(), 
server.getLocalPort());
+      try (SyncDataNodeMPPDataExchangeServiceClient held = 
manager.borrowClient(endpoint);
+          Socket peer = server.accept()) {
+        long deadline = System.currentTimeMillis() + 150;
+        assertThrows(ClientManagerException.class, () -> 
manager.borrowClient(endpoint, deadline));
+        assertTrue(System.currentTimeMillis() >= deadline - 1);
+        manager.getPool().setMaxWait(Duration.ofMillis(30));
+        assertThrows(
+            ClientManagerException.class,
+            () -> manager.borrowClient(endpoint, System.currentTimeMillis() + 
20_000));
+        assertEquals(1, manager.getPool().getNumActive());
+      }
+      assertEquals(1, manager.getPool().getNumIdle());
+    } finally {
+      manager.close();
+    }
+  }
+
+  @Test(timeout = 5_000)
+  public void stalledConnectionSetupReleasesBorrower() throws Exception {
+    MPPDataExchangeClientManager manager = new MPPDataExchangeClientManager();
+    CountDownLatch release = new CountDownLatch(1);
+    try {
+      TTransportException failure =
+          assertThrows(
+              TTransportException.class,
+              () ->
+                  manager.prepareConnection(
+                      () -> {
+                        release.await();
+                        return null;
+                      },
+                      80));
+      assertEquals(TTransportException.TIMED_OUT, failure.getType());
+      assertEquals("ready", manager.prepareConnection(() -> "ready", 1_000));
+    } finally {
+      release.countDown();
+      manager.close();
+    }
+  }
+
+  @Test
+  public void connectUsesRemainingBudgetAndClosesFailedSocket() throws 
Exception {
+    SyncDataNodeMPPDataExchangeServiceClient client =
+        new SyncDataNodeMPPDataExchangeServiceClient(
+            new 
ThriftClientProperty.Builder().setConnectionTimeoutMs(60_000).build(),
+            new TEndPoint("127.0.0.1", 10740),
+            null);
+    Socket socket = mock(Socket.class);
+    doThrow(new 
SocketTimeoutException()).when(socket).connect(any(SocketAddress.class), 
anyInt());
+    long deadline = System.currentTimeMillis() - 3_200 + 4_000;
+    assertThrows(SocketTimeoutException.class, () -> client.connect(socket, 
deadline));
+    ArgumentCaptor<Integer> timeout = ArgumentCaptor.forClass(Integer.class);
+    verify(socket).connect(any(SocketAddress.class), timeout.capture());
+    assertTrue(timeout.getValue() > 0 && timeout.getValue() <= 800);
+    verify(socket).close();
+
+    Socket expired = mock(Socket.class);
+    assertThrows(
+        QueryTimeoutException.class, () -> client.connect(expired, 
System.currentTimeMillis() - 1));
+    verify(expired, never()).connect(any(SocketAddress.class), anyInt());
+    verify(expired).close();
+  }
+}
diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift 
b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
index 8e2335fab0c..768081aad76 100644
--- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
+++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift
@@ -152,6 +152,7 @@ struct TSendFragmentInstanceReq {
   // Attempt metadata is absent for a legacy coordinator. Do not change the 
binary FI body.
   3: optional i32 queryAttemptId
   4: optional TQueryAdmissionToken admissionToken
+  5: optional i64 queryDeadlineMs
 }
 
 struct TSendFragmentInstanceResp {

Reply via email to