This is an automated email from the ASF dual-hosted git repository.

JackieTien97 pushed a commit to branch rc/2.0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 3cdfe4352a436d93e19c4a9b73320d2e50a93bf3
Author: Caideyipi <[email protected]>
AuthorDate: Tue Oct 27 15:44:35 2026 +0800

    Fix subscription commit recovery after RPC timeouts (#19023)
    
    Update ConsensusSubscriptionBroker.java
    
    * Retry explicit subscription commits after connection failures
    
    * Preserve completed subscription commit batches during retries
    
    * Preserve newer subscription checkpoints across commit retries
    
    * Retry transient subscription poll cleanup NACKs
    
    * Retry only unconfirmed subscription acknowledgements
    
    * Preserve subscription ACK progress across concurrent commits and server 
errors
    
    (cherry picked from commit 07dc28102b8840d82a7b7d7e47530f0660b4005f)
---
 .../base/AbstractSubscriptionConsumer.java         | 182 +++++++++++--
 .../base/AbstractSubscriptionProvider.java         |  40 ++-
 .../base/SubscriptionConsumerLifecycleTest.java    |   2 +-
 .../SubscriptionConsumerMultiProviderPollTest.java | 281 ---------------------
 .../db/pipe/resource/memory/PipeMemoryManager.java |   8 +-
 .../dataregion/memtable/TsFileProcessor.java       |   3 +-
 .../broker/ConsensusSubscriptionBroker.java        |   2 +-
 .../memory/PipeMemoryManagerResizeTest.java        |   4 +-
 .../storageengine/dataregion/DataRegionTest.java   |  13 +-
 .../rescon/memory/SystemInfoTest.java              |  24 +-
 10 files changed, 219 insertions(+), 340 deletions(-)

diff --git 
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
 
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
index 122b4138ee7..391c4d56e2e 100644
--- 
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
+++ 
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionConsumer.java
@@ -110,6 +110,9 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
   private static final long SLEEP_MS = 100L;
   private static final long SLEEP_DELTA_MS = 50L;
   private static final long TIMER_DELTA_MS = 250L;
+  private static final int MAX_COMMIT_RETRIES = 3;
+  private static final long COMMIT_RETRY_INTERVAL_MS =
+      ConsumerConstant.ENDPOINTS_SYNC_INTERVAL_MS_MIN_VALUE;
   private static final AtomicLong LAST_CONSUMER_INSTANCE_EPOCH = new 
AtomicLong();
 
   private final String username;
@@ -1060,9 +1063,7 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
         inFlightFilesCommitContextSet.remove(commitContext);
       }
       // construct temporary message to nack
-      nack(
-          Collections.singletonList(
-              new SubscriptionMessage(commitContext, file.getAbsolutePath(), 
null)));
+      nackAfterPollFailure(new SubscriptionMessage(commitContext, 
file.getAbsolutePath(), null), e);
       throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
     }
   }
@@ -1271,13 +1272,25 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
         throw (SubscriptionConsumerFencedException) e;
       }
       // construct temporary message to nack
-      nack(
-          Collections.singletonList(
-              new SubscriptionMessage(response.getCommitContext(), 
Collections.emptyMap())));
+      nackAfterPollFailure(
+          new SubscriptionMessage(response.getCommitContext(), 
Collections.emptyMap()), e);
       throw new SubscriptionRuntimeNonCriticalException(e.getMessage(), e);
     }
   }
 
+  private void nackAfterPollFailure(final SubscriptionMessage message, final 
Exception pollFailure)
+      throws SubscriptionException {
+    try {
+      nack(Collections.singletonList(message));
+    } catch (final SubscriptionConnectionException nackFailure) {
+      // Recovery was scheduled by commitInternal. A NACK replay can increment 
the server's poison
+      // message counter again. Preserve the poll failure and let the server 
recycle the event.
+      if (nackFailure != pollFailure) {
+        pollFailure.addSuppressed(nackFailure);
+      }
+    }
+  }
+
   private Optional<SubscriptionMessage> pollTabletsInternal(
       final SubscriptionPollResponse initialResponse, final PollTimer timer) {
     final Map<String, List<Tablet>> tablets =
@@ -1536,15 +1549,9 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
         dataNodeIdToSubscriptionCommitContexts.entrySet()) {
       final List<SubscriptionCommitContext> groupedCommitContexts = 
entry.getValue();
       requestedCount += groupedCommitContexts.size();
-      final AbstractSubscriptionProvider.CommitResult commitResp =
-          commitInternal(entry.getKey(), groupedCommitContexts, nack);
       final List<SubscriptionCommitContext> acceptedCommitContexts =
-          commitResp.getAcceptedCommitContexts();
+          commitWithRetry(entry.getKey(), groupedCommitContexts, nack);
       acceptedCount += acceptedCommitContexts.size();
-      if (!nack) {
-        overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
-        onAckedCommitContexts(acceptedCommitContexts);
-      }
       if (acceptedCommitContexts.size() != groupedCommitContexts.size()) {
         final List<SubscriptionCommitContext> failedInGroup =
             new ArrayList<>(groupedCommitContexts);
@@ -1565,6 +1572,88 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
     }
   }
 
+  private List<SubscriptionCommitContext> commitWithRetry(
+      final int dataNodeId,
+      final List<SubscriptionCommitContext> commitContexts,
+      final boolean nack)
+      throws SubscriptionException {
+    final List<SubscriptionCommitContext> remainingCommitContexts = new 
ArrayList<>(commitContexts);
+    final List<SubscriptionCommitContext> acceptedCommitContexts = new 
ArrayList<>();
+    for (int retries = 0; ; retries++) {
+      checkIfFenced();
+      AbstractSubscriptionProvider.CommitResult result =
+          AbstractSubscriptionProvider.CommitResult.empty();
+      SubscriptionConnectionException connectionFailure = null;
+      SubscriptionException failure = null;
+      try {
+        result = commitInternal(dataNodeId, remainingCommitContexts, nack);
+      } catch (final SubscriptionException e) {
+        failure = getCommitFailure(e);
+        result = getCompletedCommitResult(e);
+        if (failure instanceof SubscriptionConnectionException) {
+          connectionFailure = (SubscriptionConnectionException) failure;
+        }
+      }
+
+      final List<SubscriptionCommitContext> acceptedInAttempt = 
result.getAcceptedCommitContexts();
+      acceptedCommitContexts.addAll(acceptedInAttempt);
+      if (!nack) {
+        // Apply each response once, before waiting. Reapplying older progress 
at retry completion
+        // could overwrite a concurrent commit; accepted contexts must also 
leave the buffer now.
+        overlayCommittedPositions(result.getCommittedProgressByTopic());
+        onAckedCommitContexts(acceptedInAttempt);
+      }
+      if (failure != null && connectionFailure == null) {
+        throw failure;
+      }
+      remainingCommitContexts.removeAll(new HashSet<>(acceptedInAttempt));
+      if (nack
+          || remainingCommitContexts.isEmpty()
+          || retries >= MAX_COMMIT_RETRIES
+          || isClosed()
+          || Thread.currentThread().isInterrupted()) {
+        if (connectionFailure != null) {
+          throw connectionFailure;
+        }
+        return acceptedCommitContexts;
+      }
+      checkIfFenced();
+      // A partial response has no permanent-rejection reason. Retry only its 
rejected contexts
+      // and any batches whose response was lost, using the same budget as 
connection recovery.
+      // commitInternal has released the provider read lock before we wait for 
recovery.
+      try {
+        Thread.sleep(COMMIT_RETRY_INTERVAL_MS);
+      } catch (final InterruptedException interruptedException) {
+        Thread.currentThread().interrupt();
+        if (connectionFailure != null) {
+          connectionFailure.addSuppressed(interruptedException);
+          throw connectionFailure;
+        }
+        return acceptedCommitContexts;
+      }
+      checkIfFenced();
+      if (isClosed()) {
+        if (connectionFailure != null) {
+          throw connectionFailure;
+        }
+        return acceptedCommitContexts;
+      }
+    }
+  }
+
+  private SubscriptionException getCommitFailure(final SubscriptionException 
e) {
+    return e instanceof AbstractSubscriptionProvider.CommitException
+        ? ((AbstractSubscriptionProvider.CommitException) e).getFailure()
+        : e;
+  }
+
+  private AbstractSubscriptionProvider.CommitResult getCompletedCommitResult(
+      final SubscriptionException e) {
+    return e instanceof AbstractSubscriptionProvider.CommitException
+        ? ((AbstractSubscriptionProvider.CommitException) 
e).getCompletedResult()
+        : AbstractSubscriptionProvider.CommitResult.empty();
+  }
+
   /** Invoked after the server accepts commit contexts from an acknowledgement 
request. */
   protected void onAckedCommitContexts(
       final Collection<SubscriptionCommitContext> acceptedCommitContexts) {
@@ -1610,10 +1699,26 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
             commitInternal(entry.getKey(), groupedCommitContexts, false);
         overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
         removableCommitContexts.addAll(commitResp.getAcceptedCommitContexts());
-      } catch (final SubscriptionConnectionException e) {
+      } catch (final SubscriptionException e) {
+        final SubscriptionException failure = getCommitFailure(e);
+        final AbstractSubscriptionProvider.CommitResult completedResult =
+            getCompletedCommitResult(e);
+        
overlayCommittedPositions(completedResult.getCommittedProgressByTopic());
+        final Set<SubscriptionCommitContext> acceptedBeforeFailure =
+            new HashSet<>(completedResult.getAcceptedCommitContexts());
+        removableCommitContexts.addAll(acceptedBeforeFailure);
+        if (!(failure instanceof SubscriptionConnectionException)) {
+          // The caller cannot receive the removable set when an error 
propagates. Remove confirmed
+          // contexts from every completed group before leaving the 
automatic/close-time ACK path.
+          onAckedCommitContexts(removableCommitContexts);
+          throw failure;
+        }
         int stagedCount = 0;
         int retainedCount = 0;
         for (final SubscriptionCommitContext commitContext : 
groupedCommitContexts) {
+          if (acceptedBeforeFailure.contains(commitContext)) {
+            continue;
+          }
           if (isConsensusCommitContext(commitContext)) {
             stagePendingRedirectAck(commitContext);
             removableCommitContexts.add(commitContext);
@@ -1637,7 +1742,7 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
               this,
               retainedCount,
               entry.getKey(),
-              e);
+              failure);
         }
       }
     }
@@ -1676,6 +1781,7 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
       final List<SubscriptionCommitContext> subscriptionCommitContexts,
       final boolean nack)
       throws SubscriptionException {
+    checkIfFenced();
     providers.acquireReadLock();
     try {
       final AbstractSubscriptionProvider provider = 
providers.getProvider(dataNodeId);
@@ -1694,11 +1800,13 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
       }
       try {
         return provider.commit(subscriptionCommitContexts, nack);
-      } catch (final SubscriptionConsumerFencedException e) {
-        fence(e);
-        throw e;
-      } catch (final SubscriptionConnectionException e) {
-        providers.syncIfUnavailable(this);
+      } catch (final SubscriptionException e) {
+        final SubscriptionException failure = getCommitFailure(e);
+        if (failure instanceof SubscriptionConsumerFencedException) {
+          fence((SubscriptionConsumerFencedException) failure);
+        } else if (failure instanceof SubscriptionConnectionException) {
+          providers.syncIfUnavailable(this);
+        }
         throw e;
       }
     } finally {
@@ -2090,15 +2198,11 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
       }
 
       final List<SubscriptionCommitContext> contextsToRedirect = new 
ArrayList<>(pendingContexts);
+      AbstractSubscriptionProvider.CommitResult commitResp;
       try {
-        final AbstractSubscriptionProvider.CommitResult commitResp =
-            commitInternal(entry.getValue(), contextsToRedirect, false);
-        overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
-        
commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove);
-        if (pendingContexts.isEmpty()) {
-          pendingRedirectAcksByTopicRegion.remove(entry.getKey(), 
pendingContexts);
-        }
+        commitResp = commitInternal(entry.getValue(), contextsToRedirect, 
false);
       } catch (final SubscriptionException e) {
+        commitResp = getCompletedCommitResult(e);
         LOGGER.warn(
             SubscriptionMessages
                 
.LOG_ARG_FAILED_REDIRECT_ARG_PENDING_CONSENSUS_ACK_S_ARG_VIA_E6A10AC5,
@@ -2108,6 +2212,11 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
             entry.getValue(),
             e);
       }
+      overlayCommittedPositions(commitResp.getCommittedProgressByTopic());
+      commitResp.getAcceptedCommitContexts().forEach(pendingContexts::remove);
+      if (pendingContexts.isEmpty()) {
+        pendingRedirectAcksByTopicRegion.remove(entry.getKey(), 
pendingContexts);
+      }
     }
   }
 
@@ -2162,7 +2271,24 @@ abstract class AbstractSubscriptionConsumer implements 
AutoCloseable {
     if (Objects.isNull(progressByTopic) || progressByTopic.isEmpty()) {
       return;
     }
-    progressByTopic.forEach(this::overlayCommittedPositions);
+    // ACK responses from different providers can arrive out of order, 
including completed batches
+    // carried by a failure. Preserve the newest position for each writer. 
Explicit seek operations
+    // use the topic-scoped overlay above and can still move a checkpoint 
backward.
+    for (final Entry<String, TopicProgress> entry : 
progressByTopic.entrySet()) {
+      if (Objects.isNull(entry.getValue())) {
+        continue;
+      }
+      for (final RegionProgress regionProgress : 
entry.getValue().getRegionProgress().values()) {
+        if (Objects.nonNull(regionProgress)) {
+          regionProgress
+              .getWriterPositions()
+              .forEach(
+                  (writerId, writerProgress) ->
+                      mergeTopicProgress(
+                          committedPositionsByTopic, entry.getKey(), writerId, 
writerProgress));
+        }
+      }
+    }
   }
 
   private void overlayTopicProgress(
diff --git 
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
 
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
index 56f73249103..125005763dc 100644
--- 
a/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
+++ 
b/iotdb-client/subscription/src/main/java/org/apache/iotdb/session/subscription/consumer/base/AbstractSubscriptionProvider.java
@@ -804,20 +804,32 @@ public abstract class AbstractSubscriptionProvider {
 
     final List<SubscriptionCommitContext> acceptedCommitContexts = new 
ArrayList<>();
     final Map<String, TopicProgress> committedProgressByTopic = new 
LinkedHashMap<>();
+    int completedCommitContextCount = 0;
     rpcLock.lock();
     try {
       for (final List<SubscriptionCommitContext> batch : batches) {
         final CommitResult result = commitOnce(batch, nack);
+        completedCommitContextCount += batch.size();
         acceptedCommitContexts.addAll(result.getAcceptedCommitContexts());
         mergeTopicProgress(committedProgressByTopic, 
result.getCommittedProgressByTopic());
       }
+    } catch (final SubscriptionException e) {
+      if (nack || completedCommitContextCount == 0) {
+        throw e;
+      }
+      // Preserve every completed response even if a later batch fails with a 
server error. The
+      // consumer applies these results before propagating the original 
failure or retrying.
+      throw new CommitException(
+          e,
+          completedCommitContextCount,
+          new CommitResult(acceptedCommitContexts, committedProgressByTopic));
     } finally {
       rpcLock.unlock();
     }
     return new CommitResult(acceptedCommitContexts, committedProgressByTopic);
   }
 
-  private CommitResult commitOnce(
+  CommitResult commitOnce(
       final List<SubscriptionCommitContext> subscriptionCommitContexts, final 
boolean nack)
       throws SubscriptionException {
     final PipeSubscribeCommitReq req;
@@ -898,6 +910,32 @@ public abstract class AbstractSubscriptionProvider {
     TPipeSubscribeResp send(TPipeSubscribeReq req) throws TException;
   }
 
+  static final class CommitException extends SubscriptionException {
+    private final int completedCommitContextCount;
+    private final CommitResult completedResult;
+
+    private CommitException(
+        final SubscriptionException cause,
+        final int completedCommitContextCount,
+        final CommitResult completedResult) {
+      super(cause.getMessage(), cause);
+      this.completedCommitContextCount = completedCommitContextCount;
+      this.completedResult = completedResult;
+    }
+
+    int getCompletedCommitContextCount() {
+      return completedCommitContextCount;
+    }
+
+    CommitResult getCompletedResult() {
+      return completedResult;
+    }
+
+    SubscriptionException getFailure() {
+      return (SubscriptionException) getCause();
+    }
+  }
+
   static final class CommitResult {
     private static final CommitResult EMPTY =
         new CommitResult(Collections.emptyList(), Collections.emptyMap());
diff --git 
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java
 
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java
index 51f35c5d820..7a05318acf8 100644
--- 
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java
+++ 
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerLifecycleTest.java
@@ -436,7 +436,7 @@ public class SubscriptionConsumerLifecycleTest {
       Assert.assertEquals(1, uncommittedCommitContexts.size());
       Assert.assertEquals(
           Collections.singleton(rejectedCommitContext), 
uncommittedCommitContexts.get(0L));
-      Assert.assertEquals(1, consumer.commitRequestCount);
+      Assert.assertEquals(4, consumer.commitRequestCount);
     } finally {
       consumer.close();
     }
diff --git 
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
 
b/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
deleted file mode 100644
index 34ef7a8fc48..00000000000
--- 
a/iotdb-client/subscription/src/test/java/org/apache/iotdb/session/subscription/consumer/base/SubscriptionConsumerMultiProviderPollTest.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.session.subscription.consumer.base;
-
-import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.rpc.subscription.config.TopicConfig;
-import org.apache.iotdb.rpc.subscription.exception.SubscriptionException;
-import 
org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionCommitContext;
-import org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponse;
-import 
org.apache.iotdb.rpc.subscription.payload.poll.SubscriptionPollResponseType;
-import org.apache.iotdb.rpc.subscription.payload.poll.TabletsPayload;
-import org.apache.iotdb.rpc.subscription.payload.poll.TopicProgress;
-import 
org.apache.iotdb.rpc.subscription.payload.response.PipeSubscribeHeartbeatResp;
-import org.apache.iotdb.session.AbstractSessionBuilder;
-import org.apache.iotdb.session.subscription.SubscriptionTreeSessionBuilder;
-import org.apache.iotdb.session.subscription.payload.SubscriptionMessage;
-
-import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.write.record.Tablet;
-import org.apache.tsfile.write.schema.IMeasurementSchema;
-import org.apache.tsfile.write.schema.MeasurementSchema;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-
-public class SubscriptionConsumerMultiProviderPollTest {
-
-  private static final String HOST = "127.0.0.1";
-  private static final int FIRST_PORT = 10_001;
-  private static final String TOPIC = "topic1";
-  private static final String CONSUMER_ID = "test_consumer";
-  private static final String CONSUMER_GROUP_ID = "test_consumer_group";
-  private static final long LONG_INTERVAL_MS = 86_400_000L;
-
-  @Test
-  public void testPollTriesNextProviderBeforeBackingOff() throws 
SubscriptionException {
-    final TestPullConsumer consumer = new TestPullConsumer(2, 0);
-    try {
-      consumer.open();
-      consumer.subscribeTopic();
-
-      final List<SubscriptionMessage> messages = consumer.pollForTest(1_000L);
-
-      Assert.assertEquals(1, messages.size());
-      Assert.assertEquals(1, consumer.getPollCount(1));
-      Assert.assertEquals(1, consumer.getPollCount(2));
-      Assert.assertEquals(0, consumer.getPollCount(3));
-      Assert.assertEquals(0, consumer.getBackoffCount());
-    } finally {
-      consumer.close();
-    }
-  }
-
-  @Test
-  public void testPollBacksOffAfterAllProvidersAreEmpty() throws 
SubscriptionException {
-    final TestPullConsumer consumer = new TestPullConsumer(1, 1);
-    try {
-      consumer.open();
-      consumer.subscribeTopic();
-
-      final List<SubscriptionMessage> messages = consumer.pollForTest(1_000L);
-
-      Assert.assertEquals(1, messages.size());
-      Assert.assertEquals(2, consumer.getPollCount(1));
-      Assert.assertEquals(1, consumer.getPollCount(2));
-      Assert.assertEquals(1, consumer.getPollCount(3));
-      Assert.assertEquals(1, consumer.getBackoffCount());
-    } finally {
-      consumer.close();
-    }
-  }
-
-  private static class TestPullConsumer extends 
AbstractSubscriptionPullConsumer {
-
-    private final Map<Integer, Integer> pollCounts = new HashMap<>();
-    private final int dataProviderId;
-    private final int emptyPollsBeforeData;
-    private int backoffCount;
-
-    private TestPullConsumer(final int dataProviderId, final int 
emptyPollsBeforeData) {
-      super(
-          new AbstractSubscriptionPullConsumerBuilder()
-              .host(HOST)
-              .port(FIRST_PORT)
-              .consumerId(CONSUMER_ID)
-              .consumerGroupId(CONSUMER_GROUP_ID)
-              .heartbeatIntervalMs(LONG_INTERVAL_MS)
-              .endpointsSyncIntervalMs(LONG_INTERVAL_MS)
-              .autoCommit(false));
-      this.dataProviderId = dataProviderId;
-      this.emptyPollsBeforeData = emptyPollsBeforeData;
-    }
-
-    @Override
-    protected AbstractSubscriptionProvider constructSubscriptionProvider(
-        final TEndPoint endPoint,
-        final String username,
-        final String password,
-        final String encryptedPassword,
-        final String consumerId,
-        final String consumerGroupId,
-        final String ownerId,
-        final Long ownerEpoch,
-        final int thriftMaxFrameSize,
-        final long heartbeatIntervalMs,
-        final int connectionTimeoutInMs) {
-      return new TestSubscriptionProvider(
-          endPoint,
-          username,
-          password,
-          encryptedPassword,
-          consumerId,
-          consumerGroupId,
-          ownerId,
-          ownerEpoch,
-          thriftMaxFrameSize,
-          heartbeatIntervalMs,
-          connectionTimeoutInMs,
-          pollCounts,
-          dataProviderId,
-          emptyPollsBeforeData);
-    }
-
-    private void subscribeTopic() {
-      subscribedTopics = Collections.singletonMap(TOPIC, new TopicConfig());
-    }
-
-    private List<SubscriptionMessage> pollForTest(final long timeoutMs)
-        throws SubscriptionException {
-      return poll(timeoutMs);
-    }
-
-    private int getPollCount(final int dataNodeId) {
-      return pollCounts.getOrDefault(dataNodeId, 0);
-    }
-
-    private int getBackoffCount() {
-      return backoffCount;
-    }
-
-    @Override
-    void sleepAfterEmptyPollRound() {
-      backoffCount++;
-    }
-  }
-
-  private static class TestSubscriptionProvider extends 
AbstractSubscriptionProvider {
-
-    private final int dataNodeId;
-    private final Map<Integer, Integer> pollCounts;
-    private final int dataProviderId;
-    private final int emptyPollsBeforeData;
-
-    private TestSubscriptionProvider(
-        final TEndPoint endPoint,
-        final String username,
-        final String password,
-        final String encryptedPassword,
-        final String consumerId,
-        final String consumerGroupId,
-        final String ownerId,
-        final Long ownerEpoch,
-        final int thriftMaxFrameSize,
-        final long heartbeatIntervalMs,
-        final int connectionTimeoutInMs,
-        final Map<Integer, Integer> pollCounts,
-        final int dataProviderId,
-        final int emptyPollsBeforeData) {
-      super(
-          endPoint,
-          username,
-          password,
-          encryptedPassword,
-          consumerId,
-          consumerGroupId,
-          ownerId,
-          ownerEpoch,
-          thriftMaxFrameSize,
-          heartbeatIntervalMs,
-          connectionTimeoutInMs);
-      this.dataNodeId = endPoint.port - FIRST_PORT + 1;
-      this.pollCounts = pollCounts;
-      this.dataProviderId = dataProviderId;
-      this.emptyPollsBeforeData = emptyPollsBeforeData;
-    }
-
-    @Override
-    protected AbstractSessionBuilder constructSubscriptionSessionBuilder(
-        final String host,
-        final int port,
-        final String username,
-        final String password,
-        final String encryptedPassword,
-        final int thriftMaxFrameSize,
-        final int connectionTimeoutInMs) {
-      final boolean useEncryptedPassword = Objects.nonNull(encryptedPassword);
-      return new SubscriptionTreeSessionBuilder()
-          .host(host)
-          .port(port)
-          .username(username)
-          .password(useEncryptedPassword ? encryptedPassword : password)
-          .useEncryptedPassword(useEncryptedPassword)
-          .thriftMaxFrameSize(thriftMaxFrameSize)
-          .connectionTimeoutInMs(connectionTimeoutInMs);
-    }
-
-    @Override
-    synchronized void handshake() {
-      setAvailable();
-    }
-
-    @Override
-    synchronized void close() {
-      setUnavailable();
-    }
-
-    @Override
-    int getDataNodeId() {
-      return dataNodeId;
-    }
-
-    @Override
-    PipeSubscribeHeartbeatResp heartbeat(
-        final List<SubscriptionCommitContext> processorBufferedCommitContexts) 
{
-      final PipeSubscribeHeartbeatResp response = new 
PipeSubscribeHeartbeatResp();
-      response.getTopics().put(TOPIC, new TopicConfig());
-      response.getEndPoints().put(1, new TEndPoint(HOST, FIRST_PORT));
-      response.getEndPoints().put(2, new TEndPoint(HOST, FIRST_PORT + 1));
-      response.getEndPoints().put(3, new TEndPoint(HOST, FIRST_PORT + 2));
-      return response;
-    }
-
-    @Override
-    List<SubscriptionPollResponse> poll(
-        final Set<String> topicNames,
-        final long timeoutMs,
-        final Map<String, TopicProgress> progressByTopic)
-        throws SubscriptionException {
-      pollCounts.merge(dataNodeId, 1, Integer::sum);
-      if (dataNodeId != dataProviderId || pollCounts.get(dataNodeId) <= 
emptyPollsBeforeData) {
-        return Collections.emptyList();
-      }
-      final SubscriptionCommitContext commitContext =
-          new SubscriptionCommitContext(dataNodeId, 0, TOPIC, 
CONSUMER_GROUP_ID, 0L);
-      final List<IMeasurementSchema> schemas =
-          Collections.singletonList(new MeasurementSchema("s1", 
TSDataType.INT64));
-      final Tablet tablet = new Tablet("root.sg.d1", schemas, 1);
-      tablet.setTimestamps(new long[] {1L});
-      ((long[]) tablet.getValues()[0])[0] = 1L;
-      tablet.setRowSize(1);
-      return Collections.singletonList(
-          new SubscriptionPollResponse(
-              SubscriptionPollResponseType.TABLETS.getType(),
-              new TabletsPayload(Collections.singletonList(tablet), -1),
-              commitContext));
-    }
-  }
-}
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index 32c3eb9ed11..11c5d14aaae 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@ -855,13 +855,7 @@ public class PipeMemoryManager {
 
     if (force) {
       throw new PipeRuntimeOutOfMemoryCriticalException(
-          String.format(
-              DataNodePipeMessages
-                  
.PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC,
-              memoryAllocateMaxRetries,
-              getTotalNonFloatingMemorySizeInBytes(),
-              memoryBlock.getUsedMemoryInBytes(),
-              sizeInBytes));
+          getResizeFailureMessage(memoryAllocateMaxRetries, sizeInBytes));
     }
   }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 3601f433c25..1c0dd008dfe 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1475,7 +1475,8 @@ public class TsFileProcessor {
               break;
             }
             // The asynchronous submitter needs the region write lock held by 
this insertion.
-            // Submit other already-selected MemTables here so they can 
release memory while we wait.
+            // Submit other already-selected MemTables here so they can 
release memory while we
+            // wait.
             submitOtherSelectedMemTablesToFlush();
             try {
               
TimeUnit.MILLISECONDS.sleep(config.getCheckPeriodWhenInsertBlocked());
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java
index 8bb3b3ad084..2d9d7a58c02 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/ConsensusSubscriptionBroker.java
@@ -792,7 +792,7 @@ public class ConsensusSubscriptionBroker implements 
ISubscriptionBroker {
         topicOwnershipSnapshots.remove(topicName);
       }
     }
-    queuesToClose.forEach(queue -> queue.closeAsync(false, false));
+    queuesToClose.forEach(queue -> queue.closeAsync(false));
   }
 
   public void wakeUpPrefetchingQueues(final String topicName, final boolean 
isTableModel) {
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
index 298647b8eb3..6de1776e8be 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
@@ -259,7 +259,9 @@ public class PipeMemoryManagerResizeTest {
           Assert.assertThrows(
               PipeRuntimeOutOfMemoryCriticalException.class,
               () -> manager.forceAllocateForTsFileParserWithRetry(40));
-      
Assert.assertTrue(allocationException.getMessage().contains("usedTsFiles=1100 
bytes"));
+      Assert.assertTrue(
+          allocationException.getMessage(),
+          allocationException.getMessage().contains("usedTsFiles=1100 bytes"));
       
Assert.assertTrue(allocationException.getMessage().contains("tabletLimit=900 
bytes"));
       Assert.assertTrue(
           allocationException.getMessage().contains("tabletAndTsFileLimit=1200 
bytes"));
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
index 70810de95fa..d652b831b1d 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java
@@ -67,6 +67,7 @@ import 
org.apache.iotdb.db.storageengine.dataregion.compaction.selector.constant
 import 
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer;
 import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager;
 import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy;
+import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk;
 import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
 import 
org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate;
@@ -79,7 +80,6 @@ import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator;
 import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager;
 import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
-import org.apache.iotdb.db.storageengine.rescon.memory.WriteMemoryBlock;
 import org.apache.iotdb.db.utils.EnvironmentUtils;
 import org.apache.iotdb.db.utils.constant.TestConstant;
 import org.apache.iotdb.rpc.RpcUtils;
@@ -241,6 +241,8 @@ public class DataRegionTest {
     final boolean originalSeparateData = config.isEnableSeparateData();
     final long partitionInterval = 
TimePartitionUtils.getTimePartitionInterval();
     try {
+      systemInfo.close();
+      systemInfo.loadWriteMemory();
       config.setEnableSeparateData(true);
       for (int i = 1; i <= 384; i++) {
         insertMemoryAdmissionTestRow(i, i);
@@ -275,8 +277,7 @@ public class DataRegionTest {
       final DataRegionInfo regionInfo = (DataRegionInfo) 
regionInfoField.get(dataRegion);
       final long memoryBefore = regionInfo.getMemCost();
       setMemoryAdmissionTestThreshold("memorySizeForMemtable", 2 * 
memoryBefore);
-      setMemoryAdmissionTestThreshold(
-          "FLUSH_THRESHOLD", (double) memoryBefore - 
selected.getWorkMemTableRamCost() / 2);
+      setMemoryAdmissionTestThreshold("FLUSH_THRESHOLD", (double) memoryBefore 
- 1);
       setMemoryAdmissionTestThreshold("REJECT_THRESHOLD", (double) 
memoryBefore - 1);
       config.setMaxWaitingTimeWhenInsertBlocked(5_000);
       config.setCheckPeriodWhenInsertBlocked(10);
@@ -309,9 +310,6 @@ public class DataRegionTest {
         Assert.assertFalse(currentMemTable.shouldFlush());
         Assert.assertFalse(unselectedMemTable.shouldFlush());
         Assert.assertEquals(129, currentMemTable.getTotalPointsNum());
-        for (WriteMemoryBlock block : 
regionInfo.getWriteMemoryManager().getMemoryBlocks()) {
-          Assert.assertEquals(0, block.getReservedMemoryInBytes());
-        }
         Assert.assertFalse(systemInfo.isRejected());
       } finally {
         dataRegion.writeUnlock();
@@ -321,7 +319,7 @@ public class DataRegionTest {
       insertMemoryAdmissionTestRow(partitionInterval + 130, 130);
       Assert.assertSame(currentMemTable, current.getWorkMemTable());
       Assert.assertEquals(3, 
dataRegion.getTsFileManager().getTsFileList(true).size());
-      dataRegion.syncCloseAllWorkingAndClosingTsFileProcessors();
+      dataRegion.syncCloseAllWorkingTsFileProcessors();
       int points = 0;
       for (TsFileResource resource : 
dataRegion.getTsFileManager().getTsFileList(true)) {
         try (TsFileSequenceReader reader = new 
TsFileSequenceReader(resource.getTsFilePath());
@@ -340,6 +338,7 @@ public class DataRegionTest {
       }
       Assert.assertEquals(515, points);
     } finally {
+      systemInfo.close();
       systemInfo.loadWriteMemory();
       config.setMaxWaitingTimeWhenInsertBlocked(originalWait);
       config.setCheckPeriodWhenInsertBlocked(originalPeriod);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java
index d330af1777e..8c5e3e362d5 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfoTest.java
@@ -86,7 +86,7 @@ public class SystemInfoTest {
 
   @Test
   public void testRejectedWriterCannotFlushSmallerCurrentProcessor() throws 
Exception {
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
 
     Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
     Assert.assertTrue(systemInfo.isRejected());
@@ -97,7 +97,7 @@ public class SystemInfoTest {
   @Test
   public void testRejectedWriterFirstTabletCannotBypassSelection() throws 
Exception {
     when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L);
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
 
     Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
     verify(currentProcessor, never()).setWorkMemTableShouldFlush();
@@ -106,7 +106,7 @@ public class SystemInfoTest {
 
   @Test
   public void testRepeatedReportsSubmitEachMemtableOnce() throws Exception {
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
 
     for (int i = 0; i < 10; i++) {
       Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
@@ -120,7 +120,7 @@ public class SystemInfoTest {
   @Test
   public void testRepeatedReportsForFirstTabletSubmitOnce() throws Exception {
     when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L);
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
 
     for (int i = 0; i < 10; i++) {
       Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
@@ -134,7 +134,7 @@ public class SystemInfoTest {
   public void testNewMemtableCanBeScheduledAfterPreviousFlush() throws 
Exception {
     when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L);
     when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L);
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
     Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
 
     mockWorkMemTable(currentProcessor);
@@ -148,9 +148,9 @@ public class SystemInfoTest {
   public void testMarkedCurrentProcessorCannotReachHardMemoryLimit() throws 
Exception {
     when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L);
     when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L);
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
     Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
-    dataRegionInfo.getWriteMemoryManager().addMemory(100);
+    dataRegionInfo.addStorageGroupMemCost(100);
 
     try {
       systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor);
@@ -166,7 +166,7 @@ public class SystemInfoTest {
   public void testNoCurrentMemtableCannotBypassRejection() throws Exception {
     when(currentProcessor.getWorkMemTable()).thenReturn(null);
     when(currentProcessor.getWorkMemTableRamCost()).thenReturn(0L);
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
 
     Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
     Assert.assertTrue(systemInfo.isRejected());
@@ -176,10 +176,10 @@ public class SystemInfoTest {
 
   @Test
   public void testRejectionClearsAfterMemoryIsReleased() throws Exception {
-    dataRegionInfo.getWriteMemoryManager().addMemory(900);
+    dataRegionInfo.addStorageGroupMemCost(900);
     Assert.assertFalse(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
 
-    dataRegionInfo.getWriteMemoryManager().releaseMemory(500);
+    dataRegionInfo.releaseStorageGroupMemCost(500);
     systemInfo.resetStorageGroupStatus(dataRegionInfo);
     Assert.assertFalse(systemInfo.isRejected());
     Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
@@ -190,7 +190,7 @@ public class SystemInfoTest {
   public void testCurrentProcessorCannotExceedHardMemoryLimit() throws 
Exception {
     when(currentProcessor.getWorkMemTableRamCost()).thenReturn(700L);
     when(largerProcessor.getWorkMemTableRamCost()).thenReturn(100L);
-    dataRegionInfo.getWriteMemoryManager().addMemory(1100);
+    dataRegionInfo.addStorageGroupMemCost(1100);
 
     try {
       systemInfo.reportStorageGroupStatus(dataRegionInfo, currentProcessor);
@@ -202,7 +202,7 @@ public class SystemInfoTest {
 
   @Test
   public void testNormalFlushStillSelectsLargestMemtable() throws Exception {
-    dataRegionInfo.getWriteMemoryManager().addMemory(700);
+    dataRegionInfo.addStorageGroupMemCost(700);
 
     Assert.assertTrue(systemInfo.reportStorageGroupStatus(dataRegionInfo, 
currentProcessor));
     Assert.assertFalse(systemInfo.isRejected());

Reply via email to