This is an automated email from the ASF dual-hosted git repository.

jt2594838 pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/dev/1.3 by this push:
     new ec46eb23a95 Fix historical TsFile filtering and reference acquisition 
(#19096) (#19113)
ec46eb23a95 is described below

commit ec46eb23a955386991099d4ec04265c00b7921fe
Author: Caideyipi <[email protected]>
AuthorDate: Wed Nov 4 15:03:13 2026 +0800

    Fix historical TsFile filtering and reference acquisition (#19096) (#19113)
    
    * Fix historical TsFile filtering and reference acquisition
    
    * Clarify realtime TsFile filtering and reference ownership
    
    (cherry picked from commit 814b77028ad42035ae4a8645481d3db84a40f953)
---
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |   3 +
 .../apache/iotdb/db/i18n/DataNodePipeMessages.java |   3 +
 .../common/tsfile/PipeTsFileInsertionEvent.java    |   8 +-
 .../PipeHistoricalDataRegionTsFileSource.java      |  40 ++--
 .../pipe/event/PipeTsFileInsertionEventTest.java   |  15 ++
 .../PipeHistoricalDataRegionTsFileSourceTest.java  | 207 +++++++++++++++++++++
 6 files changed, 259 insertions(+), 17 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index db51ff09b00..02bd37fbfe6 100644
--- 
a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -23,6 +23,9 @@ public final class DataNodePipeMessages {
 
   private DataNodePipeMessages() {}
 
+  public static final String 
EXCEPTION_INCREASE_REFERENCE_COUNT_ERROR_HOLDER_MESSAGE_ARG_7E1D032D =
+      "Increase reference count error. Holder Message: %s";
+
   // ===================== LEGACY PIPE RECEIVER RESPONSES =====================
 
   public static final String MESSAGE_PIPE_DATA_TRANSPORT_ERROR_ARG_ECE6A238 =
diff --git 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
index 053fc9cf69f..8a99dd35921 100644
--- 
a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
+++ 
b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java
@@ -23,6 +23,9 @@ public final class DataNodePipeMessages {
 
   private DataNodePipeMessages() {}
 
+  public static final String 
EXCEPTION_INCREASE_REFERENCE_COUNT_ERROR_HOLDER_MESSAGE_ARG_7E1D032D =
+      "增加引用计数失败。持有者信息:%s";
+
   // ===================== LEGACY PIPE RECEIVER RESPONSES =====================
 
   public static final String MESSAGE_PIPE_DATA_TRANSPORT_ERROR_ARG_ECE6A238 = 
"Pipe 数据传输错误:%s";
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index 0d379db7831..1c76d657797 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -556,16 +556,20 @@ public class PipeTsFileInsertionEvent extends 
EnrichedEvent
 
   @Override
   public boolean mayEventTimeOverlappedWithTimeRange() {
-    // Notice that this is only called at realtime extraction, and the tsFile 
is always closed
-    // Thus we can use the end time to judge the overlap
+    // Only called during realtime extraction, after writer.endFile() has 
finalized the time bounds.
+    // The resource may not yet be marked closed, but its end time can already 
be used for overlap.
     return Objects.isNull(resource)
         || startTime <= resource.getFileEndTime() && 
resource.getFileStartTime() <= endTime;
   }
 
   @Override
   public boolean shouldParseTime() {
+    // Historical extraction may observe incomplete time bounds while an 
asynchronous close is in
+    // progress. Realtime TsFile events are published after writer.endFile(), 
when the bounds are
+    // final even if the resource's closed flag has not yet been updated by 
the close callbacks.
     if (!isTimeParsed
         && Objects.nonNull(resource)
+        && (!isGeneratedByHistoricalExtractor || resource.isClosed())
         && startTime <= resource.getFileStartTime()
         && resource.getFileEndTime() <= endTime) {
       isTimeParsed = true;
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
index 1d984f9e18f..9518253a5b7 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
@@ -35,6 +35,7 @@ import 
org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
 import 
org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSourceRuntimeEnvironment;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.PipePattern;
 import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
+import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
@@ -50,6 +51,7 @@ import 
org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeCo
 import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
 import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
 import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.exception.PipeException;
 import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException;
 
 import org.apache.tsfile.file.metadata.IDeviceID;
@@ -583,8 +585,8 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
       return false;
     }
 
-    // Some resource may not be closed due to the control of 
PIPE_MIN_FLUSH_INTERVAL_IN_MS. We
-    // simply ignore them.
+    // Closing files may still have incomplete progress and time metadata. 
Select them
+    // conservatively; the event waits for the file to close before parsing or 
transfer.
     if (!resource.isClosed()
         && Optional.ofNullable(resource.getProcessor())
             .map(TsFileProcessor::alreadyMarkedClosing)
@@ -730,7 +732,10 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
   }
 
   private boolean isTsFileResourceCoveredByTimeRange(final TsFileResource 
resource) {
-    return historicalDataExtractionStartTime <= resource.getFileStartTime()
+    // In particular, a sequence file's end time is not final until it is 
closed. Do not disable
+    // row-level filtering based on the bounds observed while an asynchronous 
close is in flight.
+    return resource.isClosed()
+        && historicalDataExtractionStartTime <= resource.getFileStartTime()
         && historicalDataExtractionEndTime >= resource.getFileEndTime();
   }
 
@@ -763,8 +768,8 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
         return supplyTerminateEvent();
       }
 
-      pendingQueue.poll();
       if (!filteredTsFileResources.contains(resource)) {
+        pendingQueue.poll();
         if (shouldUseHistoricalTsFileQueryPriorityOrder()) {
           if (shouldReportHistoricalProgressAfterResource(resource)) {
             return supplyHistoricalProgressReportEvent(
@@ -776,6 +781,7 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
       }
 
       final Event event = supplyTsFileEvent(resource);
+      pendingQueue.poll();
       if (Objects.nonNull(event) && 
shouldReportHistoricalProgressAfterResource(resource)) {
         pendingHistoricalProgressIndexToReport = 
getHistoricalProgressIndexAfterResource(resource);
       }
@@ -846,7 +852,6 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
   }
 
   private Event supplyTsFileEvent(final TsFileResource resource) {
-    filteredTsFileResources.remove(resource);
     final PipeTsFileInsertionEvent event =
         new PipeTsFileInsertionEvent(
             resource,
@@ -873,17 +878,22 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
       event.skipParsingTime();
     }
 
+    final boolean isReferenceCountIncreased =
+        
event.increaseReferenceCount(PipeHistoricalDataRegionTsFileSource.class.getName());
+    if (!isReferenceCountIncreased) {
+      // Unlike realtime supply, which already holds a queued event reference, 
this is the
+      // event's first reference acquisition. The source's pin keeps the file 
available for retry.
+      // Keep the queue entry and pin until an event owns the resource. A 
failed reference
+      // acquisition must not advance historical progress past this file.
+      throw new PipeException(
+          String.format(
+              DataNodePipeMessages
+                  
.EXCEPTION_INCREASE_REFERENCE_COUNT_ERROR_HOLDER_MESSAGE_ARG_7E1D032D,
+              event.coreReportMessage()));
+    }
+    filteredTsFileResources.remove(resource);
     try {
-      final boolean isReferenceCountIncreased =
-          
event.increaseReferenceCount(PipeHistoricalDataRegionTsFileSource.class.getName());
-      if (!isReferenceCountIncreased) {
-        LOGGER.warn(
-            "Pipe {}@{}: failed to increase reference count for historical 
event {}, will discard it",
-            pipeName,
-            dataRegionId,
-            event);
-      }
-      return isReferenceCountIncreased ? event : null;
+      return event;
     } finally {
       try {
         PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, 
pipeName);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
index b30cf4f98e3..b5a42056c7c 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
@@ -23,6 +23,7 @@ import 
org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
 import org.apache.iotdb.commons.utils.FileUtils;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex;
 
@@ -37,6 +38,20 @@ import java.nio.file.Files;
 
 public class PipeTsFileInsertionEventTest {
 
+  @Test
+  public void testRealtimeTsFileTimeParsingRemainsUnchanged() {
+    final TsFileResource resource = new TsFileResource(new 
File("1-1-0-0.tsfile"));
+    final IDeviceID device = new PlainDeviceID("root.db1.d1");
+    resource.updateStartTime(device, 0);
+    resource.updateEndTime(device, Long.MIN_VALUE);
+    resource.setStatusForTest(TsFileResourceStatus.UNCLOSED);
+    final PipeTsFileInsertionEvent event =
+        new PipeTsFileInsertionEvent(
+            resource, null, false, false, false, "realtime_time_filter", 1L, 
null, null, -10, 10);
+    Assert.assertFalse(event.shouldParseTime());
+    event.close();
+  }
+
   @Test(timeout = 60000)
   public void 
testRealtimeEventCanSkipWaitingForClosedStatusAfterTsFileSealed() throws 
Exception {
     final File tempDir = 
Files.createTempDirectory("pipeTsFileSealed").toFile();
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
index 7562e54e581..32225750a65 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
@@ -32,12 +32,17 @@ import 
org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixPipePattern;
 import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
 import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
+import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
 import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator;
 import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters;
 import org.apache.iotdb.pipe.api.event.Event;
+import org.apache.iotdb.pipe.api.exception.PipeException;
 
 import com.google.common.collect.ImmutableMap;
 import org.apache.tsfile.file.metadata.PlainDeviceID;
@@ -52,14 +57,190 @@ import java.nio.file.Files;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 public class PipeHistoricalDataRegionTsFileSourceTest {
 
+  @Test
+  public void testClosingTsFileKeepsTimeFiltering() throws Exception {
+    // A closing sequence file can temporarily look wholly inside the 
extraction window even
+    // though the eventual file also contains rows outside it.
+    assertSuppliedTsFileTimeFiltering(false, 0, Long.MIN_VALUE, false, true);
+  }
+
+  @Test
+  public void testClosedTsFileTimeFiltering() throws Exception {
+    assertSuppliedTsFileTimeFiltering(true, -10, 10, false, false);
+    assertSuppliedTsFileTimeFiltering(true, -20, 20, false, true);
+  }
+
+  @Test
+  public void testClosingTsFileHonorsLooseTimeRange() throws Exception {
+    assertSuppliedTsFileTimeFiltering(false, 0, Long.MIN_VALUE, true, false);
+  }
+
+  @Test
+  public void 
testHistoricalTsFileReferenceFailureRetriesWithoutAdvancingProgress()
+      throws Exception {
+    assertHistoricalTsFileReferenceFailure(true);
+  }
+
+  @Test
+  public void testHistoricalTsFileReferenceFailureReleasesPinOnClose() throws 
Exception {
+    assertHistoricalTsFileReferenceFailure(false);
+  }
+
+  @SuppressWarnings("unchecked")
+  private void assertHistoricalTsFileReferenceFailure(final boolean retry) 
throws Exception {
+    final File tempDir = 
Files.createTempDirectory("pipeHistoricalReferenceFailure").toFile();
+    final String pipeName = "historical_reference_retry";
+    final String resourcePipeName =
+        PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, 1L);
+    final PipeTsFileResourceManager manager = 
PipeDataNodeResourceManager.tsfile();
+    final PipeHistoricalDataRegionTsFileSource source = new 
PipeHistoricalDataRegionTsFileSource();
+    final AtomicBoolean failPathResolution = new AtomicBoolean();
+    PipeTsFileInsertionEvent event = null;
+    try {
+      final File partitionDir = new File(tempDir, 
"sequence/root.db/2147483645/0");
+      Assert.assertTrue(partitionDir.mkdirs());
+      final TsFileResource resource = createTsFileResource(partitionDir, 
"1-1-0-0.tsfile");
+      resource.setFile(new CanonicalPathFailingFile(resource.getTsFilePath(), 
failPathResolution));
+      resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+      resource.updateProgressIndex(new SimpleProgressIndex(0, 1));
+
+      setPrivateField(source, "pipeName", pipeName);
+      setPrivateField(source, "creationTime", 1L);
+      setPrivateField(source, "dataRegionId", 2147483645);
+      setPrivateField(source, "hasBeenStarted", true);
+      setPrivateField(source, "shouldExtractInsertion", true);
+      setPrivateField(source, "shouldOrderHistoricalTsFileByQueryPriority", 
true);
+      setPrivateField(source, "isDbNameCoveredByPattern", true);
+      final Set<TsFileResource> selectedResources =
+          (Set<TsFileResource>) getPrivateField(source, 
"filteredTsFileResources");
+      selectedResources.add(resource);
+      final Set<TsFileResource> progressReportResources =
+          (Set<TsFileResource>) getPrivateField(source, 
"historicalProgressReportResources");
+      progressReportResources.add(resource);
+      setPrivateField(
+          source,
+          "pendingQueue",
+          new ArrayDeque<TsFileResource>(Collections.singletonList(resource)));
+
+      manager.pinTsFileResource(resource, false, pipeName);
+      final File pinnedFile =
+          PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+              resource.getTsFile(), pipeName);
+      failPathResolution.set(true);
+      Assert.assertThrows(PipeException.class, source::supply);
+      Assert.assertEquals(1, source.getPendingQueueSize());
+      Assert.assertTrue(selectedResources.contains(resource));
+      Assert.assertTrue(progressReportResources.contains(resource));
+      Assert.assertNull(getPrivateField(source, 
"pendingHistoricalProgressIndexToReport"));
+      Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile, 
pipeName));
+
+      failPathResolution.set(false);
+      if (retry) {
+        event = (PipeTsFileInsertionEvent) source.supply();
+        Assert.assertNotNull(event);
+        final Field resourceField = 
PipeTsFileInsertionEvent.class.getDeclaredField("resource");
+        resourceField.setAccessible(true);
+        Assert.assertSame(resource, resourceField.get(event));
+        Assert.assertEquals(0, source.getPendingQueueSize());
+        Assert.assertFalse(selectedResources.contains(resource));
+        // In 1.3, the source pin and event reference use separate pipe 
directories.
+        Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile, 
pipeName));
+        Assert.assertEquals(1, 
manager.getFileReferenceCount(event.getTsFile(), resourcePipeName));
+        final Event progressEvent = source.supply();
+        Assert.assertTrue(progressEvent instanceof ProgressReportEvent);
+        Assert.assertEquals(
+            new TimePartitionProgressIndex(
+                resource.getTimePartition(), resource.getMaxProgressIndex()),
+            ((ProgressReportEvent) progressEvent).getProgressIndex());
+      } else {
+        source.close();
+        Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile, 
pipeName));
+      }
+    } finally {
+      failPathResolution.set(false);
+      if (event != null) {
+        event.clearReferenceCount(getClass().getName());
+      }
+      source.close();
+      manager.cleanPipeTsFileDir(pipeName);
+      manager.cleanPipeTsFileDir(resourcePipeName);
+      FileUtils.deleteFileOrDirectory(tempDir);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  private void assertSuppliedTsFileTimeFiltering(
+      final boolean closed,
+      final long fileStartTime,
+      final long fileEndTime,
+      final boolean looseTimeRange,
+      final boolean shouldParseTime)
+      throws Exception {
+    final File tempDir = 
Files.createTempDirectory("pipeHistoricalClosingTimeFilter").toFile();
+    final String pipeName = "closing_time_filter";
+    final String resourcePipeName =
+        PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, 1L);
+    final PipeTsFileResourceManager manager = 
PipeDataNodeResourceManager.tsfile();
+    final PipeHistoricalDataRegionTsFileSource source = new 
PipeHistoricalDataRegionTsFileSource();
+    PipeTsFileInsertionEvent event = null;
+    try {
+      final File partitionDir = new File(tempDir, 
"sequence/root.db/2147483646/0");
+      Assert.assertTrue(partitionDir.mkdirs());
+      final TsFileResource resource = createTsFileResource(partitionDir, 
"1-1-0-0.tsfile");
+      final PlainDeviceID device = new PlainDeviceID("root.db.device1");
+      resource.updateStartTime(device, fileStartTime);
+      resource.updateEndTime(device, fileEndTime);
+      resource.setStatusForTest(
+          closed ? TsFileResourceStatus.NORMAL : 
TsFileResourceStatus.UNCLOSED);
+
+      setPrivateField(source, "pipeName", pipeName);
+      setPrivateField(source, "creationTime", 1L);
+      setPrivateField(source, "dataRegionId", 2147483646);
+      setPrivateField(source, "hasBeenStarted", true);
+      setPrivateField(source, "isDbNameCoveredByPattern", true);
+      setPrivateField(source, "historicalDataExtractionStartTime", -10L);
+      setPrivateField(source, "historicalDataExtractionEndTime", 10L);
+      setPrivateField(source, "sloppyTimeRange", looseTimeRange);
+      ((Set<TsFileResource>) getPrivateField(source, 
"filteredTsFileResources")).add(resource);
+      setPrivateField(
+          source,
+          "pendingQueue",
+          new ArrayDeque<TsFileResource>(Collections.singletonList(resource)));
+      manager.pinTsFileResource(resource, false, pipeName);
+
+      event = (PipeTsFileInsertionEvent) source.supply();
+      Assert.assertNotNull(event);
+      // The collector must not send this entire file without filtering after 
its close completes.
+      Assert.assertEquals(shouldParseTime, event.shouldParseTime());
+      Assert.assertEquals(!shouldParseTime, 
PipeEventCollector.canSkipParsing4TsFileEvent(event));
+      if (!closed && !looseTimeRange) {
+        resource.updateStartTime(device, -20);
+        resource.updateEndTime(device, 20);
+        resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+        Assert.assertTrue(event.shouldParseTime());
+        
Assert.assertFalse(PipeEventCollector.canSkipParsing4TsFileEvent(event));
+      }
+    } finally {
+      if (event != null) {
+        event.clearReferenceCount(getClass().getName());
+      }
+      source.close();
+      manager.cleanPipeTsFileDir(pipeName);
+      manager.cleanPipeTsFileDir(resourcePipeName);
+      FileUtils.deleteFileOrDirectory(tempDir);
+    }
+  }
+
   @Test
   public void testGlobalTimeRangeRespectsHistoryEnable() throws Exception {
     final Map<String, String> attributes = new HashMap<>();
@@ -443,6 +624,32 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
     }
   }
 
+  private static class CanonicalPathFailingFile extends File {
+
+    private final AtomicBoolean failPathResolution;
+
+    private CanonicalPathFailingFile(final String path, final AtomicBoolean 
failPathResolution) {
+      super(path);
+      this.failPathResolution = failPathResolution;
+    }
+
+    @Override
+    public File getParentFile() {
+      final File parent = super.getParentFile();
+      return parent == null
+          ? null
+          : new CanonicalPathFailingFile(parent.getPath(), failPathResolution);
+    }
+
+    @Override
+    public String getCanonicalPath() throws IOException {
+      if (failPathResolution.get()) {
+        throw new IOException("mock path resolution failure");
+      }
+      return super.getCanonicalPath();
+    }
+  }
+
   private static void assertMayTsFileContainUnprocessedData(
       final File tempDir,
       final String fileName,

Reply via email to