This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 2cd3b07f7af [Pipe] Fix stalled region ordering after TsFile-to-Tablet 
conversion (#19243)
2cd3b07f7af is described below

commit 2cd3b07f7af82a05f4d14a7ba9dcab5cffb5115f
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 12 18:56:33 2026 +0800

    [Pipe] Fix stalled region ordering after TsFile-to-Tablet conversion 
(#19243)
    
    * [Pipe] Fix stalled region ordering after TsFile-to-Tablet conversion
    
    * [Pipe] Track processed tablet descendants for TsFile completion
---
 .../agent/task/connection/PipeEventCollector.java  |   6 +-
 .../common/tablet/PipeRawTabletInsertionEvent.java |  23 +-
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  26 +-
 .../db/pipe/source/PipeRealtimeExtractTest.java    | 311 ++++++++++++++++++++-
 4 files changed, 347 insertions(+), 19 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
index 07ed903b671..1191c1e0739 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/connection/PipeEventCollector.java
@@ -286,11 +286,11 @@ public class PipeEventCollector implements EventCollector 
{
     }
 
     if (event instanceof PipeRawTabletInsertionEvent
-        && ((PipeRawTabletInsertionEvent) event).getSourceEvent()
-            instanceof PipeTsFileInsertionEvent) {
+        && ((PipeRawTabletInsertionEvent) 
event).getSourceTsFileInsertionEvent() != null) {
       // Register before publishing to the queue so a concurrent queue cleanup 
can report a
       // generated tablet as discarded instead of racing with registration.
-      ((PipeTsFileInsertionEvent) ((PipeRawTabletInsertionEvent) 
event).getSourceEvent())
+      ((PipeRawTabletInsertionEvent) event)
+          .getSourceTsFileInsertionEvent()
           .registerGeneratedTabletInsertionEvent((PipeRawTabletInsertionEvent) 
event);
     }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
index fe8ce50130a..e41b8d39f63 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
@@ -66,6 +66,7 @@ public class PipeRawTabletInsertionEvent extends 
PipeInsertionEvent
   private final boolean isAligned;
 
   private final EnrichedEvent sourceEvent;
+  private final PipeTsFileInsertionEvent sourceTsFileInsertionEvent;
   private final AtomicBoolean generatedEventRegisteredWithSource = new 
AtomicBoolean(false);
   private final AtomicBoolean generatedEventOutcomeReported = new 
AtomicBoolean(false);
   private boolean needToReport;
@@ -115,6 +116,12 @@ public class PipeRawTabletInsertionEvent extends 
PipeInsertionEvent
     this.tablet = Objects.requireNonNull(tablet);
     this.isAligned = isAligned;
     this.sourceEvent = sourceEvent;
+    this.sourceTsFileInsertionEvent =
+        sourceEvent instanceof PipeTsFileInsertionEvent
+            ? (PipeTsFileInsertionEvent) sourceEvent
+            : sourceEvent instanceof PipeRawTabletInsertionEvent
+                ? ((PipeRawTabletInsertionEvent) 
sourceEvent).getSourceTsFileInsertionEvent()
+                : null;
     this.needToReport = needToReport;
     inheritSourceEventReportSkippingIfNecessary();
 
@@ -135,7 +142,7 @@ public class PipeRawTabletInsertionEvent extends 
PipeInsertionEvent
                 parentMemoryBlock);
     this.allocatedMemoryBlock.setAssigner(this);
 
-    if (needToReport || sourceEvent instanceof PipeTsFileInsertionEvent) {
+    if (needToReport || sourceTsFileInsertionEvent != null) {
       addOnCommittedHook(
           () -> {
             // The last generated tablet still owns the progress-index 
cleanup. Other generated
@@ -143,10 +150,9 @@ public class PipeRawTabletInsertionEvent extends 
PipeInsertionEvent
             if (shouldReportOnCommit && needToReport) {
               eliminateProgressIndex();
             }
-            if (sourceEvent instanceof PipeTsFileInsertionEvent
+            if (sourceTsFileInsertionEvent != null
                 && generatedEventOutcomeReported.compareAndSet(false, true)) {
-              ((PipeTsFileInsertionEvent) sourceEvent)
-                  .markGeneratedTabletInsertionEventAsTransferred();
+              
sourceTsFileInsertionEvent.markGeneratedTabletInsertionEventAsTransferred();
             }
           });
     }
@@ -366,8 +372,8 @@ public class PipeRawTabletInsertionEvent extends 
PipeInsertionEvent
     final boolean cleared = super.clearReferenceCount(holderMessage);
     if (generatedEventRegisteredWithSource.get()
         && generatedEventOutcomeReported.compareAndSet(false, true)
-        && sourceEvent instanceof PipeTsFileInsertionEvent) {
-      ((PipeTsFileInsertionEvent) 
sourceEvent).markGeneratedTabletInsertionEventAsDiscarded();
+        && sourceTsFileInsertionEvent != null) {
+      
sourceTsFileInsertionEvent.markGeneratedTabletInsertionEventAsDiscarded();
     }
     return cleared;
   }
@@ -494,6 +500,11 @@ public class PipeRawTabletInsertionEvent extends 
PipeInsertionEvent
     return sourceEvent;
   }
 
+  /** Returns the original TsFile, including when a processor has transformed 
its tablets. */
+  public PipeTsFileInsertionEvent getSourceTsFileInsertionEvent() {
+    return sourceTsFileInsertionEvent;
+  }
+
   @Override
   public boolean isShouldReportOnCommit() {
     return shouldReportOnCommit && needToReport;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index adc3e1d23fe..bb382721779 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -592,6 +592,18 @@ public class PipeTsFileInsertionEvent extends 
PipeInsertionEvent
     return isSuccessful;
   }
 
+  @Override
+  public synchronized boolean decreaseReferenceCount(
+      final String holderMessage, final boolean shouldReport) {
+    final boolean decreased = super.decreaseReferenceCount(holderMessage, 
shouldReport);
+    if (decreased && isReleased() && Objects.isNull(getCommitterKey())) {
+      // A split TsFile may only commit its generated tablets. Releasing that 
original event ends
+      // its processing, but the transfer hook must still wait for parsing and 
all tablet commits.
+      markAsTransferredIfGeneratedTabletEventsCompleted();
+    }
+    return decreased;
+  }
+
   @Override
   public boolean clearReferenceCount(final String holderMessage) {
     final boolean cleared = super.clearReferenceCount(holderMessage);
@@ -698,14 +710,12 @@ public class PipeTsFileInsertionEvent extends 
PipeInsertionEvent
     final boolean generationBoundaryReached =
         !generatedTabletInsertionEventsParsingStarted.get()
             || generatedTabletInsertionEventsParsingCompleted.get();
-    // The normal processor path does not retain a tablet iterable unless 
parsing has explicitly
-    // been marked as started.  In that path, registering the generated 
tablets and reaching the
-    // generation boundary is sufficient to establish the transfer boundary 
(the source TsFile
-    // commit may be reported separately or may be intentionally skipped).  
Once an iterable has
-    // been retained, however, the source TsFile commit must also be observed 
before the generated
-    // tablet counts can release the TsFile.  This distinction keeps deferred 
parsing safe without
-    // delaying the established non-deferred path.
-    if ((isTsFileEventCommitted.get() || 
!generatedTabletInsertionEventsParsingStarted.get())
+    // The collector only assigns commit ids to the generated tablets when it 
splits a TsFile.
+    // The original event then finishes processing without entering the commit 
queue. If the
+    // original event does enter that queue, resource release alone must not 
bypass its commit.
+    if ((isTsFileEventCommitted.get()
+            || (isReleased() && Objects.isNull(getCommitterKey()))
+            || !generatedTabletInsertionEventsParsingStarted.get())
         && generationBoundaryReached
         && transferredGeneratedTabletInsertionEventCount.get()
                 + discardedGeneratedTabletInsertionEventCount.get()
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
index 2bb9501f2e4..13664d88942 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipeRealtimeExtractTest.java
@@ -25,6 +25,7 @@ import 
org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
 import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.pipe.agent.task.PipeTaskAgent;
+import 
org.apache.iotdb.commons.pipe.agent.task.connection.UnboundedBlockingPendingQueue;
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMeta;
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeMetaKeeper;
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeRuntimeMeta;
@@ -41,10 +42,12 @@ import 
org.apache.iotdb.commons.queryengine.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.commons.utils.FileUtils;
 import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
+import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
 import 
org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEvent;
 import org.apache.iotdb.db.pipe.event.realtime.PipeRealtimeEventFactory;
+import org.apache.iotdb.db.pipe.metric.source.PipeDataRegionEventCounter;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionHybridSource;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionLogSource;
 import 
org.apache.iotdb.db.pipe.source.dataregion.realtime.PipeRealtimeDataRegionSource;
@@ -61,8 +64,11 @@ import 
org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
 import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent;
 
 import org.apache.tsfile.common.constant.TsFileConstant;
+import org.apache.tsfile.enums.ColumnCategory;
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.TableSchema;
+import org.apache.tsfile.write.TsFileWriter;
 import org.apache.tsfile.write.record.Tablet;
 import org.apache.tsfile.write.schema.MeasurementSchema;
 import org.junit.After;
@@ -76,10 +82,12 @@ import java.io.File;
 import java.io.IOException;
 import java.lang.reflect.Field;
 import java.nio.file.Files;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
@@ -1506,8 +1514,9 @@ public class PipeRealtimeExtractTest {
 
       final PipeTsFileInsertionEvent firstTsFile = (PipeTsFileInsertionEvent) 
extractor.supply();
       firstTsFile.markGeneratedTabletInsertionEventsParsingStarted();
-      commitSuppliedEvent(firstTsFile, commitManager);
-      // Committing a TsFile with a retained iterable and no tablets yet must 
not release the gate.
+      Assert.assertNull(firstTsFile.getCommitterKey());
+      
Assert.assertTrue(firstTsFile.decreaseReferenceCount(TEST_REFERENCE_HOLDER, 
false));
+      // Releasing the original event must not release the gate while 
generation is still open.
       Assert.assertNull(extractor.supply());
       final PipeRawTabletInsertionEvent firstTablet =
           createGeneratedTabletEvent(firstTsFile, pipeTaskMeta, "first", 
false);
@@ -1535,6 +1544,248 @@ public class PipeRealtimeExtractTest {
     }
   }
 
+  @Test
+  public void testTsFileSourceReleasesSplitTableFileAfterTabletCommit() throws 
Exception {
+    verifySplitTableFileCompletion(false);
+  }
+
+  @Test
+  public void testTsFileSourceReleasesSplitTableFileAfterOriginalRelease() 
throws Exception {
+    verifySplitTableFileCompletion(true);
+  }
+
+  private void verifySplitTableFileCompletion(final boolean 
commitTabletBeforeOriginalRelease)
+      throws Exception {
+    final PipeEventCommitManager commitManager = 
PipeEventCommitManager.getInstance();
+    commitManager.register(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
dataRegion1, "test");
+    final PipeTaskMeta pipeTaskMeta = new 
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
+    final UnboundedBlockingPendingQueue<Event> pendingQueue =
+        new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter());
+    final PipeRealtimeEvent[] events = new PipeRealtimeEvent[2];
+    try (final PipeRealtimeDataRegionTsFileSource extractor =
+        new PipeRealtimeDataRegionTsFileSource()) {
+      final PipeParameters parameters =
+          new PipeParameters(
+              Map.of(
+                  PipeSourceConstant.SOURCE_CAPTURE_TABLE_KEY, "true",
+                  PipeSourceConstant.SOURCE_TABLE_NAME_KEY, "t0",
+                  
PipeSourceConstant.SOURCE_REALTIME_REGION_LEVEL_DOWNGRADING_KEY, "true"));
+      extractor.validate(new PipeParameterValidator(parameters));
+      extractor.customize(
+          parameters,
+          new PipeTaskRuntimeConfiguration(
+              new PipeTaskSourceRuntimeEnvironment(
+                  TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, dataRegion1, 
pipeTaskMeta)));
+      for (int i = 0; i < events.length; i++) {
+        events[i] =
+            PipeRealtimeEventFactory.createRealtimeEvent(
+                    true,
+                    "table_db",
+                    createTableTsFileResource("126-" + (i + 1) + 
"-0-0.tsfile"),
+                    false)
+                .shallowCopySelfAndBindPipeTaskMetaForProgressReport(
+                    TEST_PIPE_NAME,
+                    TEST_PIPE_CREATION_TIME,
+                    pipeTaskMeta,
+                    extractor.getTreePattern(),
+                    extractor.getTablePattern(),
+                    "0",
+                    "root",
+                    "localhost",
+                    true,
+                    Long.MIN_VALUE,
+                    Long.MAX_VALUE);
+        
Assert.assertTrue(events[i].increaseReferenceCount(TEST_REFERENCE_HOLDER));
+        extractor.extract(events[i]);
+      }
+
+      final PipeTsFileInsertionEvent firstTsFile = (PipeTsFileInsertionEvent) 
extractor.supply();
+      final PipeEventCollector collector =
+          new PipeEventCollector(
+              pendingQueue,
+              TEST_PIPE_CREATION_TIME,
+              dataRegion1,
+              extractor.getSourceId(),
+              false,
+              false,
+              false);
+      collector.collect(firstTsFile);
+      Assert.assertFalse(collector.hasNoGeneratedEvent());
+      Assert.assertFalse(collector.isFailedToIncreaseReferenceCount());
+      Assert.assertNull(firstTsFile.getCommitterKey());
+      Assert.assertEquals(1, pendingQueue.size());
+      final Event generatedTablet = pendingQueue.directPoll();
+      Assert.assertTrue(generatedTablet instanceof 
PipeRawTabletInsertionEvent);
+      Assert.assertNotNull(((EnrichedEvent) 
generatedTablet).getCommitterKey());
+      try {
+        if (commitTabletBeforeOriginalRelease) {
+          Assert.assertTrue(
+              ((EnrichedEvent) generatedTablet)
+                  .decreaseReferenceCount(TEST_REFERENCE_HOLDER, true));
+          Assert.assertNull(extractor.supply());
+        }
+
+        // Mirror the processor: only the generated tablet enters the commit 
queue.
+        
Assert.assertTrue(firstTsFile.decreaseReferenceCount(TEST_REFERENCE_HOLDER, 
false));
+        if (!commitTabletBeforeOriginalRelease) {
+          Assert.assertNull(extractor.supply());
+          Assert.assertTrue(
+              ((EnrichedEvent) generatedTablet)
+                  .decreaseReferenceCount(TEST_REFERENCE_HOLDER, true));
+        }
+
+        final Event secondTsFile = extractor.supply();
+        Assert.assertSame(events[1].getEvent(), secondTsFile);
+        releaseSuppliedEvent(secondTsFile);
+      } finally {
+        ((EnrichedEvent) 
generatedTablet).clearReferenceCount(TEST_REFERENCE_HOLDER);
+      }
+    } finally {
+      pendingQueue.discardAllEvents();
+      for (final PipeRealtimeEvent event : events) {
+        if (event != null) {
+          event.clearReferenceCount(TEST_REFERENCE_HOLDER);
+        }
+      }
+      commitManager.deregister(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
dataRegion1);
+    }
+  }
+
+  @Test
+  public void testTsFileSourceWaitsForProcessedTabletCommits() throws 
Exception {
+    verifyProcessedTabletCompletion(false, false);
+  }
+
+  @Test
+  public void testTsFileSourceWaitsForDiscardedProcessedTablets() throws 
Exception {
+    verifyProcessedTabletCompletion(true, false);
+  }
+
+  @Test
+  public void 
testTsFileSourceWaitsForOriginalOrderedCommitAfterProcessedTablets()
+      throws Exception {
+    verifyProcessedTabletCompletion(true, true);
+  }
+
+  private void verifyProcessedTabletCompletion(
+      final boolean discardTablets, final boolean originalCommitPending) 
throws Exception {
+    final PipeEventCommitManager commitManager = 
PipeEventCommitManager.getInstance();
+    commitManager.register(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
dataRegion1, "test");
+    final PipeTaskMeta pipeTaskMeta = new 
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 1);
+    final UnboundedBlockingPendingQueue<Event> pendingQueue =
+        new UnboundedBlockingPendingQueue<>(new PipeDataRegionEventCounter());
+    final List<PipeRawTabletInsertionEvent> parsedTablets = new ArrayList<>();
+    final List<PipeRawTabletInsertionEvent> generatedTablets = new 
ArrayList<>();
+    final PipeRealtimeEvent[] events = new PipeRealtimeEvent[2];
+    final ProgressReportEvent earlierEvent =
+        new ProgressReportEvent(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
pipeTaskMeta);
+    try (final PipeRealtimeDataRegionTsFileSource extractor =
+        createTsFileSourceForTest(
+            PipeSourceConstant.SOURCE_REALTIME_REGION_LEVEL_DOWNGRADING_KEY, 
pipeTaskMeta)) {
+      events[0] =
+          extractTsFileForTest(
+              extractor, pipeTaskMeta, 
createTreeTsFileResource("127-1-0-0.tsfile"), false);
+      events[1] =
+          extractTsFileForTest(
+              extractor,
+              pipeTaskMeta,
+              createTsFileResource(dataRegion1, "127-2-0-0.tsfile"),
+              false);
+      final PipeTsFileInsertionEvent firstTsFile = (PipeTsFileInsertionEvent) 
extractor.supply();
+      final AtomicInteger completionCount = new AtomicInteger();
+      firstTsFile.addOnTransferredHook(completionCount::incrementAndGet);
+      if (originalCommitPending) {
+        
Assert.assertTrue(earlierEvent.increaseReferenceCount(TEST_REFERENCE_HOLDER));
+        commitManager.enrichWithCommitterKeyAndCommitId(
+            earlierEvent, TEST_PIPE_CREATION_TIME, dataRegion1);
+        commitManager.enrichWithCommitterKeyAndCommitId(
+            firstTsFile, TEST_PIPE_CREATION_TIME, dataRegion1);
+        Assert.assertNotNull(earlierEvent.getCommitterKey());
+        Assert.assertNotNull(firstTsFile.getCommitterKey());
+      } else {
+        Assert.assertNull(firstTsFile.getCommitterKey());
+      }
+      final PipeEventCollector collector =
+          new PipeEventCollector(
+              pendingQueue,
+              TEST_PIPE_CREATION_TIME,
+              dataRegion1,
+              extractor.getSourceId(),
+              false,
+              false,
+              false);
+      firstTsFile.consumeTabletInsertionEventsWithRetry(
+          parsedTablet -> {
+            
Assert.assertTrue(parsedTablet.increaseReferenceCount(TEST_REFERENCE_HOLDER));
+            parsedTablets.add(parsedTablet);
+            // A processor may produce multiple outputs from the same parsed 
tablet.
+            for (int i = 0; i < 2; i++) {
+              for (final TabletInsertionEvent processedTablet :
+                  parsedTablet.processRowByRow(
+                      (row, rowCollector) -> {
+                        try {
+                          rowCollector.collectRow(row);
+                        } catch (final IOException e) {
+                          throw new RuntimeException(e);
+                        }
+                      })) {
+                collector.collect(processedTablet);
+              }
+            }
+          },
+          TEST_REFERENCE_HOLDER);
+      Assert.assertEquals(1, parsedTablets.size());
+      Assert.assertEquals(2, pendingQueue.size());
+      Assert.assertFalse(collector.isFailedToIncreaseReferenceCount());
+      while (!pendingQueue.isEmpty()) {
+        final PipeRawTabletInsertionEvent generatedTablet =
+            (PipeRawTabletInsertionEvent) pendingQueue.directPoll();
+        Assert.assertSame(parsedTablets.get(0), 
generatedTablet.getSourceEvent());
+        Assert.assertNotNull(generatedTablet.getCommitterKey());
+        generatedTablets.add(generatedTablet);
+      }
+
+      
Assert.assertTrue(firstTsFile.decreaseReferenceCount(TEST_REFERENCE_HOLDER, 
false));
+      Assert.assertTrue(firstTsFile.isReleased());
+      Assert.assertEquals(0, completionCount.get());
+      Assert.assertNull(extractor.supply());
+      for (int i = 0; i < generatedTablets.size(); i++) {
+        final PipeRawTabletInsertionEvent generatedTablet = 
generatedTablets.get(i);
+        if (discardTablets) {
+          
Assert.assertTrue(generatedTablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+        } else {
+          
Assert.assertTrue(generatedTablet.decreaseReferenceCount(TEST_REFERENCE_HOLDER, 
true));
+        }
+        
Assert.assertFalse(generatedTablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+        if (i == 0 || originalCommitPending) {
+          Assert.assertEquals(0, completionCount.get());
+          Assert.assertNull(extractor.supply());
+        }
+      }
+      if (originalCommitPending) {
+        
Assert.assertTrue(earlierEvent.decreaseReferenceCount(TEST_REFERENCE_HOLDER, 
true));
+      }
+      Assert.assertEquals(1, completionCount.get());
+      final Event secondTsFile = extractor.supply();
+      Assert.assertSame(events[1].getEvent(), secondTsFile);
+      generatedTablets.forEach(tablet -> 
tablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+      Assert.assertEquals(1, completionCount.get());
+      Assert.assertNull(extractor.supply());
+      releaseSuppliedEvent(secondTsFile);
+    } finally {
+      pendingQueue.discardAllEvents();
+      generatedTablets.forEach(tablet -> 
tablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+      parsedTablets.forEach(tablet -> 
tablet.clearReferenceCount(TEST_REFERENCE_HOLDER));
+      for (final PipeRealtimeEvent event : events) {
+        if (event != null) {
+          event.clearReferenceCount(TEST_REFERENCE_HOLDER);
+        }
+      }
+      earlierEvent.clearReferenceCount(TEST_REFERENCE_HOLDER);
+      commitManager.deregister(TEST_PIPE_NAME, TEST_PIPE_CREATION_TIME, 
dataRegion1);
+    }
+  }
+
   @Test
   public void 
testTsFileSourceRegionLevelDowngradingCompensatesForDiscardedEvents()
       throws Exception {
@@ -1762,6 +2013,62 @@ public class PipeRealtimeExtractTest {
     return createTsFileResource(dataRegionId, "0", fileName);
   }
 
+  private TsFileResource createTreeTsFileResource(final String fileName) 
throws Exception {
+    final File dataRegionDir = new File(tsFileDir, dataRegion1 + 
File.separator + "0");
+    Assert.assertTrue(dataRegionDir.mkdirs() || dataRegionDir.isDirectory());
+    final File tsFile = new File(dataRegionDir, fileName);
+    final Tablet tablet =
+        new Tablet(
+            "root.sg.d",
+            Collections.singletonList(new MeasurementSchema("s0", 
TSDataType.INT64)),
+            1);
+    tablet.addTimestamp(0, 1);
+    tablet.addValue(0, 0, 1L);
+    try (final TsFileWriter writer = new TsFileWriter(tsFile)) {
+      writer.registerTimeseries(new PartialPath("root.sg.d"), 
tablet.getSchemas());
+      writer.writeTree(tablet);
+    }
+    final TsFileResource resource = new TsFileResource(tsFile);
+    final IDeviceID deviceId = 
IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d");
+    resource.updateStartTime(deviceId, 1);
+    resource.updateEndTime(deviceId, 1);
+    resource.close();
+    return resource;
+  }
+
+  private TsFileResource createTableTsFileResource(final String fileName) 
throws Exception {
+    final File dataRegionDir = new File(tsFileDir, dataRegion1 + 
File.separator + "0");
+    Assert.assertTrue(dataRegionDir.mkdirs() || dataRegionDir.isDirectory());
+    final File tsFile = new File(dataRegionDir, fileName);
+    final List<ColumnCategory> categories = List.of(ColumnCategory.TAG, 
ColumnCategory.FIELD);
+    try (final TsFileWriter writer = new TsFileWriter(tsFile)) {
+      writer.registerTableSchema(
+          new TableSchema(
+              "t0",
+              List.of(
+                  new MeasurementSchema("device_id", TSDataType.STRING),
+                  new MeasurementSchema("s0", TSDataType.INT64)),
+              categories));
+      final Tablet tablet =
+          new Tablet(
+              "t0",
+              List.of("device_id", "s0"),
+              List.of(TSDataType.STRING, TSDataType.INT64),
+              categories,
+              1);
+      tablet.addTimestamp(0, 1);
+      tablet.addValue(0, 0, "d0");
+      tablet.addValue(0, 1, 1L);
+      writer.writeTable(tablet);
+    }
+    final TsFileResource resource = new TsFileResource(tsFile);
+    final IDeviceID deviceId = IDeviceID.Factory.DEFAULT_FACTORY.create(new 
String[] {"t0", "d0"});
+    resource.updateStartTime(deviceId, 1);
+    resource.updateEndTime(deviceId, 1);
+    resource.close();
+    return resource;
+  }
+
   private TsFileResource createTsFileResource(
       final int dataRegionId, final String timePartition, final String 
fileName)
       throws IOException {

Reply via email to