This is an automated email from the ASF dual-hosted git repository.

JackieTien97 pushed a commit to branch rc/2.0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit ed3838227ce4aa9fa2c312faf00c1396f4a067fa
Author: Caideyipi <[email protected]>
AuthorDate: Thu Oct 22 16:29:42 2026 +0800

    [Pipe] Unify TsFile parser allocation and resize quotas (#18984)
---
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |   4 +
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |   4 +
 .../TsFileInsertionEventParserMemoryManager.java   |  20 ++-
 ...BlockType.java => PipeMemoryBlockCategory.java} |  30 +++-
 .../pipe/resource/memory/PipeMemoryBlockType.java  |   1 +
 .../db/pipe/resource/memory/PipeMemoryManager.java |  61 ++++++++
 .../memory/PipeTsFileParserMemoryBlock.java        |  27 ++++
 .../memory/PipeMemoryManagerResizeTest.java        | 169 +++++++++++++++++++++
 8 files changed, 302 insertions(+), 14 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index f3e3f5ec09d..56f39342242 100644
--- 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2378,6 +2378,10 @@ public final class DataNodePipeMessages {
   public static final String 
PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC
 =
       "forceResize: failed to allocate memory after %d retries, total memory 
size %d bytes, used "
           + "memory size %d bytes, requested memory size %d bytes";
+  public static final String 
EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E
 =
+      "forceResize failed after %d retries: total=%d bytes, used=%d bytes, "
+          + "usedTablets=%d bytes, usedTsFiles=%d bytes, tabletLimit=%d bytes, 
tsFileLimit=%d bytes, "
+          + "tabletAndTsFileLimit=%d bytes, protectedParserLimit=%d bytes, 
requested=%d bytes";
   public static final String
       
EXCEPTION_UNSUPPORTED_BATCH_TYPE_ARG_WHEN_TRANSFERRING_TABLET_INSERTION_EVENT_66153E12
 =
           "Unsupported batch type %s when transferring tablet insertion 
event.";
diff --git 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 2b963bb1c0e..f2759b87258 100644
--- 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -2210,6 +2210,10 @@ public final class DataNodePipeMessages {
       "forceAllocate:重试 %d 次后仍无法分配内存,总内存大小 %d bytes,已用内存大小 %d bytes,请求内存大小 %d 
bytes";
   public static final String 
PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC
 =
       "forceResize:重试 %d 次后仍无法分配内存,总内存大小 %d bytes,已用内存大小 %d bytes,请求内存大小 %d 
bytes";
+  public static final String 
EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E
 =
+      "forceResize 重试 %d 次后仍无法分配内存:total=%d bytes,used=%d bytes,"
+          + "usedTablets=%d bytes,usedTsFiles=%d bytes,tabletLimit=%d 
bytes,tsFileLimit=%d bytes,"
+          + "tabletAndTsFileLimit=%d bytes,protectedParserLimit=%d 
bytes,requested=%d bytes";
   public static final String
       
EXCEPTION_UNSUPPORTED_BATCH_TYPE_ARG_WHEN_TRANSFERRING_TABLET_INSERTION_EVENT_66153E12
 =
           "传输 tablet insertion event 时不支持 batch 类型 %s。";
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java
index 11a31ff77ae..7bb9a153477 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/parser/TsFileInsertionEventParserMemoryManager.java
@@ -20,9 +20,9 @@
 package org.apache.iotdb.db.pipe.event.common.tsfile.parser;
 
 import 
org.apache.iotdb.commons.exception.pipe.PipeRuntimeOutOfMemoryCriticalException;
-import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
 import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryBlock;
+import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryManager;
 
 /** Allocates parser working memory from the pool owned by the caller. */
 public interface TsFileInsertionEventParserMemoryManager {
@@ -42,12 +42,15 @@ public interface TsFileInsertionEventParserMemoryManager {
           public TsFileInsertionEventParserMemoryBlock 
forceAllocateForTabletWithRetry(
               final long sizeInBytes) {
             return new PipeBlock(
-                
PipeDataNodeResourceManager.memory().forceAllocateForTabletWithRetry(sizeInBytes));
+                PipeDataNodeResourceManager.memory()
+                    .forceAllocateForTsFileParserWithRetry(sizeInBytes));
           }
 
           @Override
           public TsFileInsertionEventParserMemoryBlock forceAllocate(final 
long sizeInBytes) {
-            return new 
PipeBlock(PipeDataNodeResourceManager.memory().forceAllocate(sizeInBytes));
+            return new PipeBlock(
+                PipeDataNodeResourceManager.memory()
+                    .forceAllocateForTsFileParserWithRetry(sizeInBytes));
           }
         };
   }
@@ -67,15 +70,10 @@ public interface TsFileInsertionEventParserMemoryManager {
     @Override
     public void forceResize(final long newSizeInBytes) {
       final long oldSize = delegate.getMemoryUsageInBytes();
-      if (!PipeDataNodeResourceManager.memory().tryResize(delegate, 
newSizeInBytes)) {
+      final PipeMemoryManager memoryManager = 
PipeDataNodeResourceManager.memory();
+      if (!memoryManager.tryResize(delegate, newSizeInBytes)) {
         throw new PipeRuntimeOutOfMemoryCriticalException(
-            String.format(
-                DataNodePipeMessages
-                    
.PIPE_EXCEPTION_FORCERESIZE_FAILED_TO_ALLOCATE_MEMORY_AFTER_D_RETRIES_TOTAL_8C6948BC,
-                0,
-                
PipeDataNodeResourceManager.memory().getTotalNonFloatingMemorySizeInBytes(),
-                
PipeDataNodeResourceManager.memory().getUsedMemorySizeInBytes(),
-                newSizeInBytes - oldSize));
+            memoryManager.getResizeFailureMessage(0, newSizeInBytes - 
oldSize));
       }
     }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java
similarity index 61%
copy from 
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
copy to 
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java
index 846fc7dd1ce..fb1e8df2c59 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockCategory.java
@@ -19,10 +19,34 @@
 
 package org.apache.iotdb.db.pipe.resource.memory;
 
-public enum PipeMemoryBlockType {
-  NORMAL,
+/** The diagnostic category of a Pipe memory block. */
+public enum PipeMemoryBlockCategory {
+  GLOBAL,
+  EVENT,
+  EVENT_CHILD,
+  PARSER,
   TABLET,
   TS_FILE,
   BATCH,
-  WAL
+  WAL,
+  CACHE,
+  RECEIVER,
+  SINK,
+  SUBSCRIPTION,
+  FLOATING,
+  OTHER;
+
+  public static PipeMemoryBlockCategory fromType(final PipeMemoryBlockType 
type) {
+    if (type == null) {
+      return OTHER;
+    }
+    return switch (type) {
+      case NORMAL -> OTHER;
+      case TABLET -> TABLET;
+      case TS_FILE_PARSER -> PARSER;
+      case TS_FILE -> TS_FILE;
+      case BATCH -> BATCH;
+      case WAL -> WAL;
+    };
+  }
 }
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
index 846fc7dd1ce..78fd9aff670 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryBlockType.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.pipe.resource.memory;
 public enum PipeMemoryBlockType {
   NORMAL,
   TABLET,
+  TS_FILE_PARSER,
   TS_FILE,
   BATCH,
   WAL
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index 6d89ac0d9d2..32c3eb9ed11 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@ -138,6 +138,34 @@ public class PipeMemoryManager {
         * totalNonFloatingMemorySizeInBytes;
   }
 
+  private boolean isEnoughForTsFileParserMemory(
+      final long extraMemoryInBytes, final long 
totalNonFloatingMemorySizeInBytes) {
+    return (double) usedMemorySizeInBytesOfTablets
+            + (double) usedMemorySizeInBytesOfTsFiles
+            + (double) extraMemoryInBytes
+        < EXCEED_PROTECT_THRESHOLD
+            * 
allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInBytes);
+  }
+
+  public synchronized String getResizeFailureMessage(final int retries, final 
long requestedSize) {
+    final long totalNonFloatingMemorySizeInBytes = 
getTotalNonFloatingMemorySizeInBytes();
+    final double combinedLimit =
+        
allowedMaxMemorySizeInBytesOfTabletsAndTsFiles(totalNonFloatingMemorySizeInBytes);
+    return String.format(
+        DataNodePipeMessages
+            
.EXCEPTION_FORCERESIZE_FAILED_AFTER_ARG_RETRIES_TOTAL_ARG_BYTES_USED_ARG_BYTES_USEDTABLETS_ARG_BYTES_USEDTSFILES_ARG_BYTES_TABLETLIMIT_ARG_BYTES_TSFILELIMIT_ARG_BYTES_TABLETANDTSFILELIMIT_ARG_BYTES_PROTECTEDPARSERLIMIT_ARG_BYTES_REQUESTED_ARG_BYTES_311C8C7E,
+        retries,
+        totalNonFloatingMemorySizeInBytes,
+        memoryBlock.getUsedMemoryInBytes(),
+        usedMemorySizeInBytesOfTablets,
+        usedMemorySizeInBytesOfTsFiles,
+        
Math.round(allowedMaxMemorySizeInBytesOfTablets(totalNonFloatingMemorySizeInBytes)),
+        
Math.round(allowedMaxMemorySizeInBytesOfTsTiles(totalNonFloatingMemorySizeInBytes)),
+        Math.round(combinedLimit),
+        Math.round(EXCEED_PROTECT_THRESHOLD * combinedLimit),
+        requestedSize);
+  }
+
   private static long getTsFileParserMemorySizeInBytes() {
     return Math.max(
         PIPE_CONFIG.getTsFileParserMemory(), 
PIPE_CONFIG.getPipeMemoryAllocateMinSizeInBytes());
@@ -542,6 +570,10 @@ public class PipeMemoryManager {
 
   private boolean isHardEnoughForResizing(
       final PipeMemoryBlock block, final long extraMemoryInBytes) {
+    if (block instanceof PipeTsFileParserMemoryBlock) {
+      return isEnoughForTsFileParserMemory(
+          extraMemoryInBytes, getTotalNonFloatingMemorySizeInBytes());
+    }
     if (!(block instanceof PipeTabletMemoryBlock) && !(block instanceof 
PipeTsFileMemoryBlock)) {
       return true;
     }
@@ -625,6 +657,27 @@ public class PipeMemoryManager {
     }
   }
 
+  public synchronized PipeTsFileParserMemoryBlock 
forceAllocateForTsFileParserWithRetry(
+      final long parserMemorySizeInBytes) {
+    if (!PIPE_MEMORY_MANAGEMENT_ENABLED) {
+      return new PipeTsFileParserMemoryBlock(0);
+    }
+
+    final PipeTsFileParserMemoryBlock block =
+        (PipeTsFileParserMemoryBlock) registerMemoryBlock(0, 
PipeMemoryBlockType.TS_FILE_PARSER);
+    if (parserMemorySizeInBytes == 0) {
+      return block;
+    }
+
+    try {
+      forceResize(block, parserMemorySizeInBytes);
+      return block;
+    } catch (final RuntimeException e) {
+      release(block);
+      throw e;
+    }
+  }
+
   public PipeTsFileMemoryBlock forceAllocateForTsFileWithRetry(long 
tsFileSizeInBytes)
       throws PipeRuntimeOutOfMemoryCriticalException {
     if (!PIPE_MEMORY_MANAGEMENT_ENABLED) {
@@ -712,6 +765,8 @@ public class PipeMemoryManager {
       switch (type) {
         case TABLET:
           return new PipeTabletMemoryBlock(sizeInBytes);
+        case TS_FILE_PARSER:
+          return new PipeTsFileParserMemoryBlock(sizeInBytes);
         case TS_FILE:
           return new PipeTsFileMemoryBlock(sizeInBytes);
         case BATCH:
@@ -811,6 +866,9 @@ public class PipeMemoryManager {
   }
 
   private boolean tryResizeInternal(final PipeMemoryBlock block, final long 
targetSize) {
+    if (block.isReleased()) {
+      return false;
+    }
     if (!PIPE_MEMORY_MANAGEMENT_ENABLED) {
       block.setMemoryUsageInBytes(targetSize);
       return true;
@@ -983,6 +1041,9 @@ public class PipeMemoryManager {
       case TABLET:
         returnedMemoryBlock = new PipeTabletMemoryBlock(sizeInBytes);
         break;
+      case TS_FILE_PARSER:
+        returnedMemoryBlock = new PipeTsFileParserMemoryBlock(sizeInBytes);
+        break;
       case TS_FILE:
         returnedMemoryBlock = new PipeTsFileMemoryBlock(sizeInBytes);
         break;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java
new file mode 100644
index 00000000000..5258d5216b5
--- /dev/null
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeTsFileParserMemoryBlock.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.iotdb.db.pipe.resource.memory;
+
+/** Parser working memory charged to TsFiles and admitted against the 
protected combined quota. */
+public class PipeTsFileParserMemoryBlock extends PipeTsFileMemoryBlock {
+
+  public PipeTsFileParserMemoryBlock(final long memoryUsageInBytes) {
+    super(memoryUsageInBytes);
+  }
+}
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
index ddaf043b969..298647b8eb3 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManagerResizeTest.java
@@ -30,6 +30,13 @@ import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 public class PipeMemoryManagerResizeTest {
@@ -226,4 +233,166 @@ public class PipeMemoryManagerResizeTest {
       manager.release(nonFloatingMemory);
     }
   }
+
+  @Test
+  public void testTsFileParserAllocationAndResizeShareProtectedCombinedLimit() 
{
+    final AtomicLong floatingMemoryUsageInBytes = new AtomicLong(0);
+    final PipeMemoryManager manager =
+        new PipeMemoryManager(
+            new AtomicLongMemoryBlock(
+                "PipeMemoryManagerResizeTest",
+                null,
+                TOTAL_MEMORY_SIZE_IN_BYTES,
+                MemoryBlockType.DYNAMIC),
+            floatingMemoryUsageInBytes::get);
+    final PipeTsFileParserMemoryBlock parser = 
manager.forceAllocateForTsFileParserWithRetry(1000);
+
+    try {
+      Assert.assertTrue(manager.tryResize(parser, 1100));
+      Assert.assertFalse(manager.tryResize(parser, 1140));
+      Assert.assertEquals(1100, parser.getMemoryUsageInBytes());
+      Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets());
+      Assert.assertEquals(1100, manager.getUsedMemorySizeInBytesOfTsFiles());
+      Assert.assertEquals(1100, manager.getUsedMemorySizeInBytes());
+
+      final PipeRuntimeOutOfMemoryCriticalException allocationException =
+          Assert.assertThrows(
+              PipeRuntimeOutOfMemoryCriticalException.class,
+              () -> manager.forceAllocateForTsFileParserWithRetry(40));
+      
Assert.assertTrue(allocationException.getMessage().contains("usedTsFiles=1100 
bytes"));
+      
Assert.assertTrue(allocationException.getMessage().contains("tabletLimit=900 
bytes"));
+      Assert.assertTrue(
+          allocationException.getMessage().contains("tabletAndTsFileLimit=1200 
bytes"));
+      Assert.assertTrue(
+          allocationException.getMessage().contains("protectedParserLimit=1140 
bytes"));
+
+      // Parser working memory must not consume the tablet-only quota: the 
generated tablet
+      // still needs to be retained and sent while the parser owns its decoded 
pages/chunks.
+      final PipeTabletMemoryBlock output = 
manager.forceAllocateForTabletWithRetry(50);
+      try {
+        Assert.assertTrue(manager.tryResize(output, 90));
+        Assert.assertEquals(90, manager.getUsedMemorySizeInBytesOfTablets());
+        Assert.assertEquals(1100, manager.getUsedMemorySizeInBytesOfTsFiles());
+      } finally {
+        manager.release(output);
+      }
+    } finally {
+      manager.release(parser);
+    }
+
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytes());
+  }
+
+  @Test
+  public void testZeroSizedTsFileParserCanGrowBeyondTabletLimitAndRelease() {
+    final PipeMemoryManager manager =
+        new PipeMemoryManager(
+            new AtomicLongMemoryBlock(
+                "PipeMemoryManagerResizeTest",
+                null,
+                TOTAL_MEMORY_SIZE_IN_BYTES,
+                MemoryBlockType.DYNAMIC),
+            () -> 0);
+    final PipeTsFileParserMemoryBlock parser = 
manager.forceAllocateForTsFileParserWithRetry(0);
+    try {
+      Assert.assertTrue(manager.tryResize(parser, 1000));
+      Assert.assertEquals(1000, manager.getUsedMemorySizeInBytesOfTsFiles());
+      Assert.assertEquals(1000, manager.getUsedMemorySizeInBytes());
+
+      Assert.assertTrue(manager.tryResize(parser, 100));
+      Assert.assertEquals(100, manager.getUsedMemorySizeInBytesOfTsFiles());
+      manager.release(parser);
+      Assert.assertTrue(parser.isReleased());
+      Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles());
+      Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets());
+      Assert.assertEquals(0, manager.getUsedMemorySizeInBytes());
+    } finally {
+      manager.release(parser);
+    }
+  }
+
+  @Test
+  public void 
testTsFileParserSharesQuotaWithTabletsAndTsFilesAndHonorsGlobalLimit() {
+    final PipeMemoryManager manager =
+        new PipeMemoryManager(
+            new AtomicLongMemoryBlock(
+                "PipeMemoryManagerResizeTest",
+                null,
+                TOTAL_MEMORY_SIZE_IN_BYTES,
+                MemoryBlockType.DYNAMIC),
+            () -> 0);
+    final PipeTabletMemoryBlock tablet = 
manager.forceAllocateForTabletWithRetry(200);
+    final PipeTsFileMemoryBlock tsFile = 
manager.forceAllocateForTsFileWithRetry(500);
+    final PipeTsFileParserMemoryBlock parser = 
manager.forceAllocateForTsFileParserWithRetry(400);
+    final PipeMemoryBlock sink = manager.forceAllocate(900);
+    try {
+      Assert.assertFalse(manager.tryResize(parser, 440));
+      Assert.assertFalse(manager.tryResize(parser, 401));
+      Assert.assertEquals(200, manager.getUsedMemorySizeInBytesOfTablets());
+      Assert.assertEquals(900, manager.getUsedMemorySizeInBytesOfTsFiles());
+      Assert.assertEquals(2000, manager.getUsedMemorySizeInBytes());
+
+      manager.release(sink);
+      Assert.assertTrue(manager.tryResize(parser, 439));
+      Assert.assertFalse(manager.tryResize(parser, 440));
+      Assert.assertEquals(200, manager.getUsedMemorySizeInBytesOfTablets());
+      Assert.assertEquals(939, manager.getUsedMemorySizeInBytesOfTsFiles());
+      Assert.assertEquals(1139, manager.getUsedMemorySizeInBytes());
+    } finally {
+      manager.release(tablet);
+      manager.release(tsFile);
+      manager.release(parser);
+      manager.release(sink);
+    }
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets());
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles());
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytes());
+  }
+
+  @Test
+  public void testConcurrentTsFileParserAllocationsCannotOvershootQuota() 
throws Exception {
+    final PipeMemoryManager manager =
+        new PipeMemoryManager(
+            new AtomicLongMemoryBlock(
+                "PipeMemoryManagerResizeTest",
+                null,
+                TOTAL_MEMORY_SIZE_IN_BYTES,
+                MemoryBlockType.DYNAMIC),
+            () -> 0);
+    final ExecutorService executor = Executors.newFixedThreadPool(4);
+    final CountDownLatch start = new CountDownLatch(1);
+    final List<Future<PipeTsFileParserMemoryBlock>> futures = new 
ArrayList<>();
+    final List<PipeTsFileParserMemoryBlock> allocated = new ArrayList<>();
+    try {
+      for (int i = 0; i < 20; i++) {
+        futures.add(
+            executor.submit(
+                () -> {
+                  start.await();
+                  try {
+                    return manager.forceAllocateForTsFileParserWithRetry(300);
+                  } catch (final PipeRuntimeOutOfMemoryCriticalException e) {
+                    return null;
+                  }
+                }));
+      }
+      start.countDown();
+      for (final Future<PipeTsFileParserMemoryBlock> future : futures) {
+        final PipeTsFileParserMemoryBlock block = future.get(10, 
TimeUnit.SECONDS);
+        if (block != null) {
+          allocated.add(block);
+        }
+      }
+      Assert.assertEquals(3, allocated.size());
+      Assert.assertEquals(900, manager.getUsedMemorySizeInBytes());
+      Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets());
+      Assert.assertEquals(900, manager.getUsedMemorySizeInBytesOfTsFiles());
+    } finally {
+      executor.shutdownNow();
+      allocated.forEach(manager::release);
+    }
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytes());
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTsFiles());
+    Assert.assertEquals(0, manager.getUsedMemorySizeInBytesOfTablets());
+  }
 }

Reply via email to