This is an automated email from the ASF dual-hosted git repository.

Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/dev/1.3 by this push:
     new 846bf99a510 [To dev/1.3] [Pipe] Preserve historical TsFile mods after 
compaction (#18904) (#18923)
846bf99a510 is described below

commit 846bf99a510fbdd02869c307febdc0758c18b338
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 14:26:17 2026 +0800

    [To dev/1.3] [Pipe] Preserve historical TsFile mods after compaction 
(#18904) (#18923)
    
    * [To dev/1.3] [Pipe] Preserve historical TsFile mods after compaction 
(#18904)
    
    Backport only the historical TsFile/mod reference fix. Adapt to the 1.3 
ModificationFile API and use the same creation-time-qualified pipe directory 
for historical selection and events.
    
    (cherry picked from commit 10c5f89ba52ec39cd6bfb6b8b735287cc0d935f9)
    
    * Update PipeHistoricalDataRegionTsFileSourceTest.java
---
 .../common/tsfile/PipeTsFileInsertionEvent.java    |  18 +++-
 .../resource/tsfile/PipeTsFileResourceManager.java |  38 ++++----
 .../PipeHistoricalDataRegionTsFileSource.java      |  22 ++++-
 .../pipe/event/PipeTsFileInsertionEventTest.java   | 104 ++++++++++++++++++---
 .../resource/PipeTsFileResourceManagerTest.java    |  48 ++++++++++
 .../PipeHistoricalDataRegionTsFileSourceTest.java  |  83 ++++++++++++++--
 6 files changed, 273 insertions(+), 40 deletions(-)

diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index a434d2679bf..8eb2fa79bd2 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -190,7 +190,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
     this.tsFile = Objects.isNull(tsFile) ? resource.getTsFile() : tsFile;
 
     final ModificationFile modFile = resource.getModFile();
-    this.isWithMod = isWithMod && modFile.exists();
+    // Historical files and their mods have already been pinned. Compaction 
may have removed
+    // the original resource before this event is created, so inspect the 
pinned mods later.
+    this.isWithMod = isWithMod && (isGeneratedByHistoricalExtractor || 
modFile.exists());
     this.modFile = this.isWithMod ? new File(modFile.getFilePath()) : null;
 
     this.isLoaded = isLoaded;
@@ -365,9 +367,22 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
         PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, 
creationTime);
     final File originalTsFile = tsFile;
     final File originalModFile = modFile;
+    final boolean originalIsWithMod = isWithMod;
     File increasedTsFile = null;
     boolean increased = false;
     try {
+      if (isGeneratedByHistoricalExtractor) {
+        tsFile =
+            PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+                tsFile, pipeTsFileResourcePipeName);
+        if (isWithMod) {
+          modFile = new File(tsFile + ModificationFile.FILE_SUFFIX);
+          isWithMod = modFile.exists();
+          if (!isWithMod) {
+            modFile = null;
+          }
+        }
+      }
       increasedTsFile =
           PipeDataNodeResourceManager.tsfile()
               .increaseFileReference(tsFile, true, pipeTsFileResourcePipeName);
@@ -390,6 +405,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
       }
       tsFile = originalTsFile;
       modFile = originalModFile;
+      isWithMod = originalIsWithMod;
       LOGGER.warn(
           String.format(
               "Increase reference count for TsFile %s or modFile %s error. 
Holder Message: %s",
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index 0dfefda76e4..542d610d6fe 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -427,7 +427,8 @@ public class PipeTsFileResourceManager {
         : hardlinkOrCopiedFileToTsFilePublicResourceMap;
   }
 
-  public void pinTsFileResource(
+  /** Returns whether a companion mod-file reference was acquired. */
+  public boolean pinTsFileResource(
       final TsFileResource resource, final boolean withMods, final @Nullable 
String pipeName)
       throws IOException {
     final File pinnedTsFile = increaseFileReference(resource.getTsFile(), 
true, pipeName);
@@ -438,7 +439,9 @@ public class PipeTsFileResourceManager {
         synchronized (modFile) {
           increaseFileReference(new File(modFile.getFilePath()), false, 
pipeName);
         }
+        return true;
       }
+      return false;
     } catch (final IOException | RuntimeException e) {
       // Pinning is a two-file operation.  Do not leave the TsFile pinned when 
the mod file cannot
       // be pinned (for example, when the pipe directory is temporarily 
unavailable).
@@ -451,7 +454,8 @@ public class PipeTsFileResourceManager {
     }
   }
 
-  public void unpinTsFileResource(final TsFileResource resource, final 
@Nullable String pipeName)
+  public void unpinTsFileResource(
+      final TsFileResource resource, final @Nullable String pipeName, final 
boolean pinnedModFile)
       throws IOException {
     Exception firstException = null;
     File pinnedFile = null;
@@ -464,20 +468,22 @@ public class PipeTsFileResourceManager {
 
     // Always attempt the mod-file cleanup even when resolving/decreasing the 
TsFile fails.  A
     // failed first cleanup must not strand the second reference.
-    try {
-      final File pinnedModFile =
-          pinnedFile != null
-              ? new File(pinnedFile + ModificationFile.FILE_SUFFIX)
-              : getHardlinkOrCopiedFileInPipeDir(
-                  new File(resource.getModFile().getFilePath()), pipeName);
-      if (pinnedModFile.exists()) {
-        decreaseFileReference(pinnedModFile, pipeName);
-      }
-    } catch (final IOException | RuntimeException e) {
-      if (firstException == null) {
-        firstException = e;
-      } else {
-        firstException.addSuppressed(e);
+    // Use the state captured when pinning. Compaction can remove the original 
mod file, and
+    // an event can acquire mod-file references even when no mod file was 
originally pinned.
+    if (pinnedModFile) {
+      try {
+        decreaseFileReference(
+            pinnedFile != null
+                ? new File(pinnedFile + ModificationFile.FILE_SUFFIX)
+                : getHardlinkOrCopiedFileInPipeDir(
+                    new File(resource.getModFile().getFilePath()), pipeName),
+            pipeName);
+      } catch (final IOException | RuntimeException e) {
+        if (firstException == null) {
+          firstException = e;
+        } else {
+          firstException.addSuppressed(e);
+        }
       }
     }
 
diff --git 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
index 9518253a5b7..8fff0e77609 100644
--- 
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
+++ 
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages;
 import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
 import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter;
 import org.apache.iotdb.db.storageengine.StorageEngine;
 import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
@@ -111,6 +112,7 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
 
   private String pipeName;
   private long creationTime;
+  private String pipeNameWithCreationTime;
 
   private PipeTaskMeta pipeTaskMeta;
   private ProgressIndex startIndex;
@@ -143,6 +145,7 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
 
   private Queue<TsFileResource> pendingQueue;
   private final Set<TsFileResource> filteredTsFileResources = new HashSet<>();
+  private final Set<TsFileResource> resourcesWithPinnedModFile = new 
HashSet<>();
   private final Set<TsFileResource> historicalProgressReportResources = new 
HashSet<>();
   private ProgressIndex maxHistoricalProgressIndex = 
MinimumProgressIndex.INSTANCE;
   private ProgressIndex maxSuppliedHistoricalProgressReportIndex = 
MinimumProgressIndex.INSTANCE;
@@ -295,6 +298,8 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
 
     pipeName = environment.getPipeName();
     creationTime = environment.getCreationTime();
+    pipeNameWithCreationTime =
+        PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, 
creationTime);
     pipeTaskMeta = environment.getPipeTaskMeta();
     startIndex = environment.getPipeTaskMeta().getProgressIndex();
 
@@ -426,8 +431,10 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
               // Pin the resource, in case the file is removed by compaction 
or anything.
               // Will unpin it after the PipeTsFileInsertionEvent is created 
and pinned.
               try {
-                PipeDataNodeResourceManager.tsfile()
-                    .pinTsFileResource(resource, shouldTransferModFile, 
pipeName);
+                if (PipeDataNodeResourceManager.tsfile()
+                    .pinTsFileResource(resource, shouldTransferModFile, 
pipeNameWithCreationTime)) {
+                  resourcesWithPinnedModFile.add(resource);
+                }
                 return false;
               } catch (final IOException e) {
                 LOGGER.warn("Pipe: failed to pin TsFileResource {}", 
resource.getTsFilePath(), e);
@@ -896,7 +903,9 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
       return event;
     } finally {
       try {
-        PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, 
pipeName);
+        PipeDataNodeResourceManager.tsfile()
+            .unpinTsFileResource(
+                resource, pipeNameWithCreationTime, 
resourcesWithPinnedModFile.remove(resource));
       } catch (final IOException e) {
         LOGGER.warn(
             "Pipe {}@{}: failed to unpin TsFileResource after creating event, 
original path: {}",
@@ -929,7 +938,11 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
       pendingQueue.forEach(
           resource -> {
             try {
-              
PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
+              PipeDataNodeResourceManager.tsfile()
+                  .unpinTsFileResource(
+                      resource,
+                      pipeNameWithCreationTime,
+                      resourcesWithPinnedModFile.remove(resource));
             } catch (final IOException e) {
               LOGGER.warn(
                   "Pipe {}@{}: failed to unpin TsFileResource after dropping 
pipe, original path: {}",
@@ -941,6 +954,7 @@ public class PipeHistoricalDataRegionTsFileSource 
implements PipeHistoricalDataR
       pendingQueue.clear();
       pendingQueue = null;
     }
+    resourcesWithPinnedModFile.clear();
     historicalProgressReportResources.clear();
     pendingHistoricalProgressIndexToReport = null;
   }
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
index b5a42056c7c..c8098e49810 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
@@ -20,8 +20,13 @@
 package org.apache.iotdb.db.pipe.event;
 
 import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
+import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.utils.FileUtils;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
@@ -35,21 +40,98 @@ import org.junit.Test;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
 
 public class PipeTsFileInsertionEventTest {
 
   @Test
-  public void testRealtimeTsFileTimeParsingRemainsUnchanged() {
-    final TsFileResource resource = new TsFileResource(new 
File("1-1-0-0.tsfile"));
-    final IDeviceID device = new PlainDeviceID("root.db1.d1");
-    resource.updateStartTime(device, 0);
-    resource.updateEndTime(device, Long.MIN_VALUE);
-    resource.setStatusForTest(TsFileResourceStatus.UNCLOSED);
-    final PipeTsFileInsertionEvent event =
-        new PipeTsFileInsertionEvent(
-            resource, null, false, false, false, "realtime_time_filter", 1L, 
null, null, -10, 10);
-    Assert.assertFalse(event.shouldParseTime());
-    event.close();
+  public void testHistoricalModsSurviveOriginalFileRemoval() throws Exception {
+    assertHistoricalModsAfterOriginalFileRemoval(true, true);
+  }
+
+  @Test
+  public void testHistoricalFileWithoutModsSurvivesOriginalFileRemoval() 
throws Exception {
+    assertHistoricalModsAfterOriginalFileRemoval(false, true);
+  }
+
+  @Test
+  public void testHistoricalModsRemainDisabledAfterOriginalFileRemoval() 
throws Exception {
+    assertHistoricalModsAfterOriginalFileRemoval(true, false);
+  }
+
+  private void assertHistoricalModsAfterOriginalFileRemoval(
+      final boolean hasMods, final boolean transferMods) throws Exception {
+    final File tempDir = 
Files.createTempDirectory("pipeHistoricalMods").toFile();
+    final String pipeName = "historical_mods";
+    final long creationTime = 1L;
+    final String resourcePipeName =
+        PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, 
creationTime);
+    final PipeTsFileResourceManager manager = 
PipeDataNodeResourceManager.tsfile();
+    PipeTsFileInsertionEvent event = null;
+    boolean increased = false;
+    File pinnedTsFile = null;
+    File pinnedModFile = null;
+    try {
+      final File dataDir = new File(tempDir, "unsequence/root.db/0/0");
+      Assert.assertTrue(dataDir.mkdirs());
+      final TsFileResource resource =
+          createNonEmptyTsFileResource(dataDir, "1-1-0-0.tsfile", 1L, 0);
+      resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+      final Deletion deletion = new Deletion(new PartialPath("root.db.d0.s1"), 
0, 1L, 1L);
+      if (hasMods) {
+        resource.getModFile().write(deletion);
+        resource.getModFile().close();
+      }
+      manager.pinTsFileResource(resource, transferMods, resourcePipeName);
+      pinnedTsFile =
+          PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+              resource.getTsFile(), resourcePipeName);
+      pinnedModFile = new File(pinnedTsFile + ModificationFile.FILE_SUFFIX);
+
+      // Compaction removes the original after historical selection has pinned 
both files.
+      Assert.assertTrue(resource.remove());
+      Assert.assertFalse(resource.modFileExists());
+      Assert.assertEquals(hasMods && transferMods, pinnedModFile.exists());
+      event =
+          new PipeTsFileInsertionEvent(
+              resource,
+              null,
+              transferMods,
+              false,
+              true,
+              pipeName,
+              creationTime,
+              null,
+              null,
+              Long.MIN_VALUE,
+              Long.MAX_VALUE);
+      increased = 
event.internallyIncreaseResourceReferenceCount(getClass().getName());
+      Assert.assertTrue(increased);
+      Assert.assertEquals(hasMods && transferMods, event.isWithMod());
+      if (hasMods && transferMods) {
+        Assert.assertEquals(pinnedModFile, event.getModFile());
+        try (final ModificationFile mods = new 
ModificationFile(event.getModFile().getPath())) {
+          Assert.assertEquals(
+              Collections.singletonList(deletion), new 
ArrayList<>(mods.getModifications()));
+        }
+      } else {
+        Assert.assertNull(event.getModFile());
+      }
+    } finally {
+      if (increased) {
+        event.internallyDecreaseResourceReferenceCount(getClass().getName());
+      } else if (event != null) {
+        event.close();
+      }
+      if (pinnedModFile != null) {
+        manager.decreaseFileReference(pinnedModFile, resourcePipeName);
+      }
+      if (pinnedTsFile != null) {
+        manager.decreaseFileReference(pinnedTsFile, resourcePipeName);
+      }
+      FileUtils.deleteFileOrDirectory(tempDir);
+    }
   }
 
   @Test(timeout = 60000)
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
index 06f11ed9b76..07b245b233e 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
@@ -28,6 +28,7 @@ import 
org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
 import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
 import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 
 import org.apache.tsfile.enums.TSDataType;
 import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -239,6 +240,53 @@ public class PipeTsFileResourceManagerTest {
     Assert.assertFalse(Files.exists(originModFile.toPath()));
   }
 
+  @Test
+  public void testUnpinAfterOriginalModsRemoved() throws Exception {
+    final TsFileResource resource = new TsFileResource(new File(TS_FILE_NAME));
+    final boolean pinnedModFile =
+        pipeTsFileResourceManager.pinTsFileResource(resource, true, PIPE_NAME);
+    Assert.assertTrue(pinnedModFile);
+    final File pipeTsFile =
+        
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile(),
 PIPE_NAME);
+    final File pipeMods = new File(pipeTsFile + ModificationFile.FILE_SUFFIX);
+    Assert.assertEquals(1, 
pipeTsFileResourceManager.getFileReferenceCount(pipeMods, PIPE_NAME));
+
+    Assert.assertTrue(resource.remove());
+    Assert.assertFalse(resource.modFileExists());
+    Assert.assertTrue(pipeMods.exists());
+    pipeTsFileResourceManager.unpinTsFileResource(resource, PIPE_NAME, 
pinnedModFile);
+
+    Assert.assertEquals(0, 
pipeTsFileResourceManager.getFileReferenceCount(pipeTsFile, PIPE_NAME));
+    Assert.assertEquals(0, 
pipeTsFileResourceManager.getFileReferenceCount(pipeMods, PIPE_NAME));
+    Assert.assertEquals(0, 
pipeTsFileResourceManager.getFileReferenceCount(pipeMods, null));
+    Assert.assertFalse(pipeMods.exists());
+  }
+
+  @Test
+  public void testUnpinDoesNotReleaseModsCreatedAfterPinning() throws 
Exception {
+    final TsFileResource resource = new TsFileResource(new File(TS_FILE_NAME));
+    resource.removeModFile();
+    final boolean pinnedModFile =
+        pipeTsFileResourceManager.pinTsFileResource(resource, true, PIPE_NAME);
+    Assert.assertFalse(pinnedModFile);
+
+    // An event can acquire mods created after historical selection pinned the 
TsFile alone.
+    resource
+        .getModFile()
+        .write(new Deletion(new PartialPath("root.lemming.**"), 0, 0, 
Long.MAX_VALUE));
+    resource.getModFile().close();
+    final File eventMods =
+        pipeTsFileResourceManager.increaseFileReference(
+            new File(resource.getModFile().getFilePath()), false, PIPE_NAME);
+    pipeTsFileResourceManager.unpinTsFileResource(resource, PIPE_NAME, 
pinnedModFile);
+
+    Assert.assertEquals(1, 
pipeTsFileResourceManager.getFileReferenceCount(eventMods, PIPE_NAME));
+    Assert.assertTrue(eventMods.exists());
+    pipeTsFileResourceManager.decreaseFileReference(eventMods, PIPE_NAME);
+    Assert.assertEquals(0, 
pipeTsFileResourceManager.getFileReferenceCount(eventMods, PIPE_NAME));
+    Assert.assertFalse(eventMods.exists());
+  }
+
   @Test
   public void testConcurrentIncreaseTsFile() throws Exception {
     assertConcurrentIncreaseFileReference(new File(TS_FILE_NAME), true);
diff --git 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
index 32225750a65..29942d75104 100644
--- 
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
+++ 
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
@@ -26,9 +26,11 @@ import 
org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
 import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex;
 import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
 import 
org.apache.iotdb.commons.consensus.index.impl.TimePartitionProgressIndex;
+import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
 import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
 import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
+import 
org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSourceRuntimeEnvironment;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixPipePattern;
 import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
 import org.apache.iotdb.commons.utils.FileUtils;
@@ -36,6 +38,8 @@ import 
org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
 import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import 
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
 import 
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
@@ -67,6 +71,67 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 public class PipeHistoricalDataRegionTsFileSourceTest {
 
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testHistoricalSourceSuppliesPinnedModsAfterOriginalFileRemoval() 
throws Exception {
+    final File tempDir = 
Files.createTempDirectory("pipeHistoricalSourceMods").toFile();
+    final PipeHistoricalDataRegionTsFileSource source = new 
PipeHistoricalDataRegionTsFileSource();
+    final String pipeName = "historical_source_mods";
+    final long creationTime = 1L;
+    final String resourcePipeName =
+        PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, 
creationTime);
+    final PipeTsFileResourceManager manager = 
PipeDataNodeResourceManager.tsfile();
+    PipeTsFileInsertionEvent event = null;
+    try {
+      final PipeParameters parameters =
+          new PipeParameters(
+              new HashMap<>(
+                  Collections.singletonMap(
+                      PipeSourceConstant.SOURCE_MODS_ENABLE_KEY, 
Boolean.TRUE.toString())));
+      source.validate(new PipeParameterValidator(parameters));
+      source.customize(
+          parameters,
+          () ->
+              new PipeTaskSourceRuntimeEnvironment(
+                  pipeName, creationTime, 0, new 
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 0)));
+      Assert.assertEquals(resourcePipeName, getPrivateField(source, 
"pipeNameWithCreationTime"));
+
+      final File dataDir = new File(tempDir, "unsequence/root.db/0/0");
+      Assert.assertTrue(dataDir.mkdirs());
+      final TsFileResource resource =
+          createClosedTsFileResourceWithDevices(dataDir, "1-1-0-0.tsfile", 
"root.db.d0");
+      resource.getModFile().write(new Deletion(new 
PartialPath("root.db.d0.s1"), 0, 0, 1));
+      resource.getModFile().close();
+      Assert.assertTrue(manager.pinTsFileResource(resource, true, 
resourcePipeName));
+      ((Set<TsFileResource>) getPrivateField(source, 
"resourcesWithPinnedModFile")).add(resource);
+      ((Set<TsFileResource>) getPrivateField(source, 
"filteredTsFileResources")).add(resource);
+      setPrivateField(source, "pendingQueue", new 
ArrayDeque<>(Collections.singleton(resource)));
+      setPrivateField(source, "hasBeenStarted", true);
+      setPrivateField(source, "sloppyPattern", true);
+      setPrivateField(source, "sloppyTimeRange", true);
+
+      Assert.assertTrue(resource.remove());
+      event = (PipeTsFileInsertionEvent) source.supply();
+      Assert.assertNotNull(event);
+      Assert.assertTrue(event.isWithMod());
+      final File pinnedTsFile =
+          PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+              resource.getTsFile(), resourcePipeName);
+      Assert.assertEquals(pinnedTsFile, event.getTsFile());
+      Assert.assertEquals(
+          new File(pinnedTsFile + ModificationFile.FILE_SUFFIX), 
event.getModFile());
+      Assert.assertEquals(1, manager.getFileReferenceCount(event.getModFile(), 
resourcePipeName));
+      Assert.assertTrue(event.getModFile().exists());
+    } finally {
+      source.close();
+      if (event != null) {
+        event.clearReferenceCount(getClass().getName());
+      }
+      manager.cleanPipeTsFileDir(resourcePipeName);
+      FileUtils.deleteFileOrDirectory(tempDir);
+    }
+  }
+
   @Test
   public void testClosingTsFileKeepsTimeFiltering() throws Exception {
     // A closing sequence file can temporarily look wholly inside the 
extraction window even
@@ -116,6 +181,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
 
       setPrivateField(source, "pipeName", pipeName);
       setPrivateField(source, "creationTime", 1L);
+      setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName);
       setPrivateField(source, "dataRegionId", 2147483645);
       setPrivateField(source, "hasBeenStarted", true);
       setPrivateField(source, "shouldExtractInsertion", true);
@@ -132,17 +198,17 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
           "pendingQueue",
           new ArrayDeque<TsFileResource>(Collections.singletonList(resource)));
 
-      manager.pinTsFileResource(resource, false, pipeName);
+      manager.pinTsFileResource(resource, false, resourcePipeName);
       final File pinnedFile =
           PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
-              resource.getTsFile(), pipeName);
+              resource.getTsFile(), resourcePipeName);
       failPathResolution.set(true);
       Assert.assertThrows(PipeException.class, source::supply);
       Assert.assertEquals(1, source.getPendingQueueSize());
       Assert.assertTrue(selectedResources.contains(resource));
       Assert.assertTrue(progressReportResources.contains(resource));
       Assert.assertNull(getPrivateField(source, 
"pendingHistoricalProgressIndexToReport"));
-      Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile, 
pipeName));
+      Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile, 
resourcePipeName));
 
       failPathResolution.set(false);
       if (retry) {
@@ -153,9 +219,8 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
         Assert.assertSame(resource, resourceField.get(event));
         Assert.assertEquals(0, source.getPendingQueueSize());
         Assert.assertFalse(selectedResources.contains(resource));
-        // In 1.3, the source pin and event reference use separate pipe 
directories.
-        Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile, 
pipeName));
-        Assert.assertEquals(1, 
manager.getFileReferenceCount(event.getTsFile(), resourcePipeName));
+        Assert.assertEquals(pinnedFile, event.getTsFile());
+        Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile, 
resourcePipeName));
         final Event progressEvent = source.supply();
         Assert.assertTrue(progressEvent instanceof ProgressReportEvent);
         Assert.assertEquals(
@@ -164,7 +229,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
             ((ProgressReportEvent) progressEvent).getProgressIndex());
       } else {
         source.close();
-        Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile, 
pipeName));
+        Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile, 
resourcePipeName));
       }
     } finally {
       failPathResolution.set(false);
@@ -205,6 +270,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
 
       setPrivateField(source, "pipeName", pipeName);
       setPrivateField(source, "creationTime", 1L);
+      setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName);
       setPrivateField(source, "dataRegionId", 2147483646);
       setPrivateField(source, "hasBeenStarted", true);
       setPrivateField(source, "isDbNameCoveredByPattern", true);
@@ -216,7 +282,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
           source,
           "pendingQueue",
           new ArrayDeque<TsFileResource>(Collections.singletonList(resource)));
-      manager.pinTsFileResource(resource, false, pipeName);
+      manager.pinTsFileResource(resource, false, resourcePipeName);
 
       event = (PipeTsFileInsertionEvent) source.supply();
       Assert.assertNotNull(event);
@@ -237,6 +303,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
       source.close();
       manager.cleanPipeTsFileDir(pipeName);
       manager.cleanPipeTsFileDir(resourcePipeName);
+
       FileUtils.deleteFileOrDirectory(tempDir);
     }
   }

Reply via email to