This is an automated email from the ASF dual-hosted git repository.
Caideyipi pushed a commit to branch dev/1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/dev/1.3 by this push:
new 846bf99a510 [To dev/1.3] [Pipe] Preserve historical TsFile mods after
compaction (#18904) (#18923)
846bf99a510 is described below
commit 846bf99a510fbdd02869c307febdc0758c18b338
Author: Caideyipi <[email protected]>
AuthorDate: Thu Nov 5 14:26:17 2026 +0800
[To dev/1.3] [Pipe] Preserve historical TsFile mods after compaction
(#18904) (#18923)
* [To dev/1.3] [Pipe] Preserve historical TsFile mods after compaction
(#18904)
Backport only the historical TsFile/mod reference fix. Adapt to the 1.3
ModificationFile API and use the same creation-time-qualified pipe directory
for historical selection and events.
(cherry picked from commit 10c5f89ba52ec39cd6bfb6b8b735287cc0d935f9)
* Update PipeHistoricalDataRegionTsFileSourceTest.java
---
.../common/tsfile/PipeTsFileInsertionEvent.java | 18 +++-
.../resource/tsfile/PipeTsFileResourceManager.java | 38 ++++----
.../PipeHistoricalDataRegionTsFileSource.java | 22 ++++-
.../pipe/event/PipeTsFileInsertionEventTest.java | 104 ++++++++++++++++++---
.../resource/PipeTsFileResourceManagerTest.java | 48 ++++++++++
.../PipeHistoricalDataRegionTsFileSourceTest.java | 83 ++++++++++++++--
6 files changed, 273 insertions(+), 40 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
index a434d2679bf..8eb2fa79bd2 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/PipeTsFileInsertionEvent.java
@@ -190,7 +190,9 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
this.tsFile = Objects.isNull(tsFile) ? resource.getTsFile() : tsFile;
final ModificationFile modFile = resource.getModFile();
- this.isWithMod = isWithMod && modFile.exists();
+ // Historical files and their mods have already been pinned. Compaction
may have removed
+ // the original resource before this event is created, so inspect the
pinned mods later.
+ this.isWithMod = isWithMod && (isGeneratedByHistoricalExtractor ||
modFile.exists());
this.modFile = this.isWithMod ? new File(modFile.getFilePath()) : null;
this.isLoaded = isLoaded;
@@ -365,9 +367,22 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName,
creationTime);
final File originalTsFile = tsFile;
final File originalModFile = modFile;
+ final boolean originalIsWithMod = isWithMod;
File increasedTsFile = null;
boolean increased = false;
try {
+ if (isGeneratedByHistoricalExtractor) {
+ tsFile =
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ tsFile, pipeTsFileResourcePipeName);
+ if (isWithMod) {
+ modFile = new File(tsFile + ModificationFile.FILE_SUFFIX);
+ isWithMod = modFile.exists();
+ if (!isWithMod) {
+ modFile = null;
+ }
+ }
+ }
increasedTsFile =
PipeDataNodeResourceManager.tsfile()
.increaseFileReference(tsFile, true, pipeTsFileResourcePipeName);
@@ -390,6 +405,7 @@ public class PipeTsFileInsertionEvent extends EnrichedEvent
}
tsFile = originalTsFile;
modFile = originalModFile;
+ isWithMod = originalIsWithMod;
LOGGER.warn(
String.format(
"Increase reference count for TsFile %s or modFile %s error.
Holder Message: %s",
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
index 0dfefda76e4..542d610d6fe 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/tsfile/PipeTsFileResourceManager.java
@@ -427,7 +427,8 @@ public class PipeTsFileResourceManager {
: hardlinkOrCopiedFileToTsFilePublicResourceMap;
}
- public void pinTsFileResource(
+ /** Returns whether a companion mod-file reference was acquired. */
+ public boolean pinTsFileResource(
final TsFileResource resource, final boolean withMods, final @Nullable
String pipeName)
throws IOException {
final File pinnedTsFile = increaseFileReference(resource.getTsFile(),
true, pipeName);
@@ -438,7 +439,9 @@ public class PipeTsFileResourceManager {
synchronized (modFile) {
increaseFileReference(new File(modFile.getFilePath()), false,
pipeName);
}
+ return true;
}
+ return false;
} catch (final IOException | RuntimeException e) {
// Pinning is a two-file operation. Do not leave the TsFile pinned when
the mod file cannot
// be pinned (for example, when the pipe directory is temporarily
unavailable).
@@ -451,7 +454,8 @@ public class PipeTsFileResourceManager {
}
}
- public void unpinTsFileResource(final TsFileResource resource, final
@Nullable String pipeName)
+ public void unpinTsFileResource(
+ final TsFileResource resource, final @Nullable String pipeName, final
boolean pinnedModFile)
throws IOException {
Exception firstException = null;
File pinnedFile = null;
@@ -464,20 +468,22 @@ public class PipeTsFileResourceManager {
// Always attempt the mod-file cleanup even when resolving/decreasing the
TsFile fails. A
// failed first cleanup must not strand the second reference.
- try {
- final File pinnedModFile =
- pinnedFile != null
- ? new File(pinnedFile + ModificationFile.FILE_SUFFIX)
- : getHardlinkOrCopiedFileInPipeDir(
- new File(resource.getModFile().getFilePath()), pipeName);
- if (pinnedModFile.exists()) {
- decreaseFileReference(pinnedModFile, pipeName);
- }
- } catch (final IOException | RuntimeException e) {
- if (firstException == null) {
- firstException = e;
- } else {
- firstException.addSuppressed(e);
+ // Use the state captured when pinning. Compaction can remove the original
mod file, and
+ // an event can acquire mod-file references even when no mod file was
originally pinned.
+ if (pinnedModFile) {
+ try {
+ decreaseFileReference(
+ pinnedFile != null
+ ? new File(pinnedFile + ModificationFile.FILE_SUFFIX)
+ : getHardlinkOrCopiedFileInPipeDir(
+ new File(resource.getModFile().getFilePath()), pipeName),
+ pipeName);
+ } catch (final IOException | RuntimeException e) {
+ if (firstException == null) {
+ firstException = e;
+ } else {
+ firstException.addSuppressed(e);
+ }
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
index 9518253a5b7..8fff0e77609 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSource.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.db.i18n.DataNodePipeMessages;
import org.apache.iotdb.db.pipe.event.common.terminate.PipeTerminateEvent;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
import org.apache.iotdb.db.pipe.source.dataregion.DataRegionListeningFilter;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
@@ -111,6 +112,7 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
private String pipeName;
private long creationTime;
+ private String pipeNameWithCreationTime;
private PipeTaskMeta pipeTaskMeta;
private ProgressIndex startIndex;
@@ -143,6 +145,7 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
private Queue<TsFileResource> pendingQueue;
private final Set<TsFileResource> filteredTsFileResources = new HashSet<>();
+ private final Set<TsFileResource> resourcesWithPinnedModFile = new
HashSet<>();
private final Set<TsFileResource> historicalProgressReportResources = new
HashSet<>();
private ProgressIndex maxHistoricalProgressIndex =
MinimumProgressIndex.INSTANCE;
private ProgressIndex maxSuppliedHistoricalProgressReportIndex =
MinimumProgressIndex.INSTANCE;
@@ -295,6 +298,8 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
pipeName = environment.getPipeName();
creationTime = environment.getCreationTime();
+ pipeNameWithCreationTime =
+ PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName,
creationTime);
pipeTaskMeta = environment.getPipeTaskMeta();
startIndex = environment.getPipeTaskMeta().getProgressIndex();
@@ -426,8 +431,10 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
// Pin the resource, in case the file is removed by compaction
or anything.
// Will unpin it after the PipeTsFileInsertionEvent is created
and pinned.
try {
- PipeDataNodeResourceManager.tsfile()
- .pinTsFileResource(resource, shouldTransferModFile,
pipeName);
+ if (PipeDataNodeResourceManager.tsfile()
+ .pinTsFileResource(resource, shouldTransferModFile,
pipeNameWithCreationTime)) {
+ resourcesWithPinnedModFile.add(resource);
+ }
return false;
} catch (final IOException e) {
LOGGER.warn("Pipe: failed to pin TsFileResource {}",
resource.getTsFilePath(), e);
@@ -896,7 +903,9 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
return event;
} finally {
try {
- PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource,
pipeName);
+ PipeDataNodeResourceManager.tsfile()
+ .unpinTsFileResource(
+ resource, pipeNameWithCreationTime,
resourcesWithPinnedModFile.remove(resource));
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after creating event,
original path: {}",
@@ -929,7 +938,11 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
pendingQueue.forEach(
resource -> {
try {
-
PipeDataNodeResourceManager.tsfile().unpinTsFileResource(resource, pipeName);
+ PipeDataNodeResourceManager.tsfile()
+ .unpinTsFileResource(
+ resource,
+ pipeNameWithCreationTime,
+ resourcesWithPinnedModFile.remove(resource));
} catch (final IOException e) {
LOGGER.warn(
"Pipe {}@{}: failed to unpin TsFileResource after dropping
pipe, original path: {}",
@@ -941,6 +954,7 @@ public class PipeHistoricalDataRegionTsFileSource
implements PipeHistoricalDataR
pendingQueue.clear();
pendingQueue = null;
}
+ resourcesWithPinnedModFile.clear();
historicalProgressReportResources.clear();
pendingHistoricalProgressIndexToReport = null;
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
index b5a42056c7c..c8098e49810 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTsFileInsertionEventTest.java
@@ -20,8 +20,13 @@
package org.apache.iotdb.db.pipe.event;
import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
+import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
+import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
@@ -35,21 +40,98 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collections;
public class PipeTsFileInsertionEventTest {
@Test
- public void testRealtimeTsFileTimeParsingRemainsUnchanged() {
- final TsFileResource resource = new TsFileResource(new
File("1-1-0-0.tsfile"));
- final IDeviceID device = new PlainDeviceID("root.db1.d1");
- resource.updateStartTime(device, 0);
- resource.updateEndTime(device, Long.MIN_VALUE);
- resource.setStatusForTest(TsFileResourceStatus.UNCLOSED);
- final PipeTsFileInsertionEvent event =
- new PipeTsFileInsertionEvent(
- resource, null, false, false, false, "realtime_time_filter", 1L,
null, null, -10, 10);
- Assert.assertFalse(event.shouldParseTime());
- event.close();
+ public void testHistoricalModsSurviveOriginalFileRemoval() throws Exception {
+ assertHistoricalModsAfterOriginalFileRemoval(true, true);
+ }
+
+ @Test
+ public void testHistoricalFileWithoutModsSurvivesOriginalFileRemoval()
throws Exception {
+ assertHistoricalModsAfterOriginalFileRemoval(false, true);
+ }
+
+ @Test
+ public void testHistoricalModsRemainDisabledAfterOriginalFileRemoval()
throws Exception {
+ assertHistoricalModsAfterOriginalFileRemoval(true, false);
+ }
+
+ private void assertHistoricalModsAfterOriginalFileRemoval(
+ final boolean hasMods, final boolean transferMods) throws Exception {
+ final File tempDir =
Files.createTempDirectory("pipeHistoricalMods").toFile();
+ final String pipeName = "historical_mods";
+ final long creationTime = 1L;
+ final String resourcePipeName =
+ PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName,
creationTime);
+ final PipeTsFileResourceManager manager =
PipeDataNodeResourceManager.tsfile();
+ PipeTsFileInsertionEvent event = null;
+ boolean increased = false;
+ File pinnedTsFile = null;
+ File pinnedModFile = null;
+ try {
+ final File dataDir = new File(tempDir, "unsequence/root.db/0/0");
+ Assert.assertTrue(dataDir.mkdirs());
+ final TsFileResource resource =
+ createNonEmptyTsFileResource(dataDir, "1-1-0-0.tsfile", 1L, 0);
+ resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+ final Deletion deletion = new Deletion(new PartialPath("root.db.d0.s1"),
0, 1L, 1L);
+ if (hasMods) {
+ resource.getModFile().write(deletion);
+ resource.getModFile().close();
+ }
+ manager.pinTsFileResource(resource, transferMods, resourcePipeName);
+ pinnedTsFile =
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ resource.getTsFile(), resourcePipeName);
+ pinnedModFile = new File(pinnedTsFile + ModificationFile.FILE_SUFFIX);
+
+ // Compaction removes the original after historical selection has pinned
both files.
+ Assert.assertTrue(resource.remove());
+ Assert.assertFalse(resource.modFileExists());
+ Assert.assertEquals(hasMods && transferMods, pinnedModFile.exists());
+ event =
+ new PipeTsFileInsertionEvent(
+ resource,
+ null,
+ transferMods,
+ false,
+ true,
+ pipeName,
+ creationTime,
+ null,
+ null,
+ Long.MIN_VALUE,
+ Long.MAX_VALUE);
+ increased =
event.internallyIncreaseResourceReferenceCount(getClass().getName());
+ Assert.assertTrue(increased);
+ Assert.assertEquals(hasMods && transferMods, event.isWithMod());
+ if (hasMods && transferMods) {
+ Assert.assertEquals(pinnedModFile, event.getModFile());
+ try (final ModificationFile mods = new
ModificationFile(event.getModFile().getPath())) {
+ Assert.assertEquals(
+ Collections.singletonList(deletion), new
ArrayList<>(mods.getModifications()));
+ }
+ } else {
+ Assert.assertNull(event.getModFile());
+ }
+ } finally {
+ if (increased) {
+ event.internallyDecreaseResourceReferenceCount(getClass().getName());
+ } else if (event != null) {
+ event.close();
+ }
+ if (pinnedModFile != null) {
+ manager.decreaseFileReference(pinnedModFile, resourcePipeName);
+ }
+ if (pinnedTsFile != null) {
+ manager.decreaseFileReference(pinnedTsFile, resourcePipeName);
+ }
+ FileUtils.deleteFileOrDirectory(tempDir);
+ }
}
@Test(timeout = 60000)
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
index 06f11ed9b76..07b245b233e 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java
@@ -28,6 +28,7 @@ import
org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
@@ -239,6 +240,53 @@ public class PipeTsFileResourceManagerTest {
Assert.assertFalse(Files.exists(originModFile.toPath()));
}
+ @Test
+ public void testUnpinAfterOriginalModsRemoved() throws Exception {
+ final TsFileResource resource = new TsFileResource(new File(TS_FILE_NAME));
+ final boolean pinnedModFile =
+ pipeTsFileResourceManager.pinTsFileResource(resource, true, PIPE_NAME);
+ Assert.assertTrue(pinnedModFile);
+ final File pipeTsFile =
+
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(resource.getTsFile(),
PIPE_NAME);
+ final File pipeMods = new File(pipeTsFile + ModificationFile.FILE_SUFFIX);
+ Assert.assertEquals(1,
pipeTsFileResourceManager.getFileReferenceCount(pipeMods, PIPE_NAME));
+
+ Assert.assertTrue(resource.remove());
+ Assert.assertFalse(resource.modFileExists());
+ Assert.assertTrue(pipeMods.exists());
+ pipeTsFileResourceManager.unpinTsFileResource(resource, PIPE_NAME,
pinnedModFile);
+
+ Assert.assertEquals(0,
pipeTsFileResourceManager.getFileReferenceCount(pipeTsFile, PIPE_NAME));
+ Assert.assertEquals(0,
pipeTsFileResourceManager.getFileReferenceCount(pipeMods, PIPE_NAME));
+ Assert.assertEquals(0,
pipeTsFileResourceManager.getFileReferenceCount(pipeMods, null));
+ Assert.assertFalse(pipeMods.exists());
+ }
+
+ @Test
+ public void testUnpinDoesNotReleaseModsCreatedAfterPinning() throws
Exception {
+ final TsFileResource resource = new TsFileResource(new File(TS_FILE_NAME));
+ resource.removeModFile();
+ final boolean pinnedModFile =
+ pipeTsFileResourceManager.pinTsFileResource(resource, true, PIPE_NAME);
+ Assert.assertFalse(pinnedModFile);
+
+ // An event can acquire mods created after historical selection pinned the
TsFile alone.
+ resource
+ .getModFile()
+ .write(new Deletion(new PartialPath("root.lemming.**"), 0, 0,
Long.MAX_VALUE));
+ resource.getModFile().close();
+ final File eventMods =
+ pipeTsFileResourceManager.increaseFileReference(
+ new File(resource.getModFile().getFilePath()), false, PIPE_NAME);
+ pipeTsFileResourceManager.unpinTsFileResource(resource, PIPE_NAME,
pinnedModFile);
+
+ Assert.assertEquals(1,
pipeTsFileResourceManager.getFileReferenceCount(eventMods, PIPE_NAME));
+ Assert.assertTrue(eventMods.exists());
+ pipeTsFileResourceManager.decreaseFileReference(eventMods, PIPE_NAME);
+ Assert.assertEquals(0,
pipeTsFileResourceManager.getFileReferenceCount(eventMods, PIPE_NAME));
+ Assert.assertFalse(eventMods.exists());
+ }
+
@Test
public void testConcurrentIncreaseTsFile() throws Exception {
assertConcurrentIncreaseFileReference(new File(TS_FILE_NAME), true);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
index 32225750a65..29942d75104 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileSourceTest.java
@@ -26,9 +26,11 @@ import
org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.RecoverProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex;
import
org.apache.iotdb.commons.consensus.index.impl.TimePartitionProgressIndex;
+import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
+import
org.apache.iotdb.commons.pipe.config.plugin.env.PipeTaskSourceRuntimeEnvironment;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixPipePattern;
import org.apache.iotdb.commons.pipe.event.ProgressReportEvent;
import org.apache.iotdb.commons.utils.FileUtils;
@@ -36,6 +38,8 @@ import
org.apache.iotdb.db.pipe.agent.task.connection.PipeEventCollector;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager;
+import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
+import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
@@ -67,6 +71,67 @@ import java.util.concurrent.atomic.AtomicBoolean;
public class PipeHistoricalDataRegionTsFileSourceTest {
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testHistoricalSourceSuppliesPinnedModsAfterOriginalFileRemoval()
throws Exception {
+ final File tempDir =
Files.createTempDirectory("pipeHistoricalSourceMods").toFile();
+ final PipeHistoricalDataRegionTsFileSource source = new
PipeHistoricalDataRegionTsFileSource();
+ final String pipeName = "historical_source_mods";
+ final long creationTime = 1L;
+ final String resourcePipeName =
+ PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName,
creationTime);
+ final PipeTsFileResourceManager manager =
PipeDataNodeResourceManager.tsfile();
+ PipeTsFileInsertionEvent event = null;
+ try {
+ final PipeParameters parameters =
+ new PipeParameters(
+ new HashMap<>(
+ Collections.singletonMap(
+ PipeSourceConstant.SOURCE_MODS_ENABLE_KEY,
Boolean.TRUE.toString())));
+ source.validate(new PipeParameterValidator(parameters));
+ source.customize(
+ parameters,
+ () ->
+ new PipeTaskSourceRuntimeEnvironment(
+ pipeName, creationTime, 0, new
PipeTaskMeta(MinimumProgressIndex.INSTANCE, 0)));
+ Assert.assertEquals(resourcePipeName, getPrivateField(source,
"pipeNameWithCreationTime"));
+
+ final File dataDir = new File(tempDir, "unsequence/root.db/0/0");
+ Assert.assertTrue(dataDir.mkdirs());
+ final TsFileResource resource =
+ createClosedTsFileResourceWithDevices(dataDir, "1-1-0-0.tsfile",
"root.db.d0");
+ resource.getModFile().write(new Deletion(new
PartialPath("root.db.d0.s1"), 0, 0, 1));
+ resource.getModFile().close();
+ Assert.assertTrue(manager.pinTsFileResource(resource, true,
resourcePipeName));
+ ((Set<TsFileResource>) getPrivateField(source,
"resourcesWithPinnedModFile")).add(resource);
+ ((Set<TsFileResource>) getPrivateField(source,
"filteredTsFileResources")).add(resource);
+ setPrivateField(source, "pendingQueue", new
ArrayDeque<>(Collections.singleton(resource)));
+ setPrivateField(source, "hasBeenStarted", true);
+ setPrivateField(source, "sloppyPattern", true);
+ setPrivateField(source, "sloppyTimeRange", true);
+
+ Assert.assertTrue(resource.remove());
+ event = (PipeTsFileInsertionEvent) source.supply();
+ Assert.assertNotNull(event);
+ Assert.assertTrue(event.isWithMod());
+ final File pinnedTsFile =
+ PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
+ resource.getTsFile(), resourcePipeName);
+ Assert.assertEquals(pinnedTsFile, event.getTsFile());
+ Assert.assertEquals(
+ new File(pinnedTsFile + ModificationFile.FILE_SUFFIX),
event.getModFile());
+ Assert.assertEquals(1, manager.getFileReferenceCount(event.getModFile(),
resourcePipeName));
+ Assert.assertTrue(event.getModFile().exists());
+ } finally {
+ source.close();
+ if (event != null) {
+ event.clearReferenceCount(getClass().getName());
+ }
+ manager.cleanPipeTsFileDir(resourcePipeName);
+ FileUtils.deleteFileOrDirectory(tempDir);
+ }
+ }
+
@Test
public void testClosingTsFileKeepsTimeFiltering() throws Exception {
// A closing sequence file can temporarily look wholly inside the
extraction window even
@@ -116,6 +181,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
setPrivateField(source, "pipeName", pipeName);
setPrivateField(source, "creationTime", 1L);
+ setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName);
setPrivateField(source, "dataRegionId", 2147483645);
setPrivateField(source, "hasBeenStarted", true);
setPrivateField(source, "shouldExtractInsertion", true);
@@ -132,17 +198,17 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
"pendingQueue",
new ArrayDeque<TsFileResource>(Collections.singletonList(resource)));
- manager.pinTsFileResource(resource, false, pipeName);
+ manager.pinTsFileResource(resource, false, resourcePipeName);
final File pinnedFile =
PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir(
- resource.getTsFile(), pipeName);
+ resource.getTsFile(), resourcePipeName);
failPathResolution.set(true);
Assert.assertThrows(PipeException.class, source::supply);
Assert.assertEquals(1, source.getPendingQueueSize());
Assert.assertTrue(selectedResources.contains(resource));
Assert.assertTrue(progressReportResources.contains(resource));
Assert.assertNull(getPrivateField(source,
"pendingHistoricalProgressIndexToReport"));
- Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile,
pipeName));
+ Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile,
resourcePipeName));
failPathResolution.set(false);
if (retry) {
@@ -153,9 +219,8 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
Assert.assertSame(resource, resourceField.get(event));
Assert.assertEquals(0, source.getPendingQueueSize());
Assert.assertFalse(selectedResources.contains(resource));
- // In 1.3, the source pin and event reference use separate pipe
directories.
- Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile,
pipeName));
- Assert.assertEquals(1,
manager.getFileReferenceCount(event.getTsFile(), resourcePipeName));
+ Assert.assertEquals(pinnedFile, event.getTsFile());
+ Assert.assertEquals(1, manager.getFileReferenceCount(pinnedFile,
resourcePipeName));
final Event progressEvent = source.supply();
Assert.assertTrue(progressEvent instanceof ProgressReportEvent);
Assert.assertEquals(
@@ -164,7 +229,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
((ProgressReportEvent) progressEvent).getProgressIndex());
} else {
source.close();
- Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile,
pipeName));
+ Assert.assertEquals(0, manager.getFileReferenceCount(pinnedFile,
resourcePipeName));
}
} finally {
failPathResolution.set(false);
@@ -205,6 +270,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
setPrivateField(source, "pipeName", pipeName);
setPrivateField(source, "creationTime", 1L);
+ setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName);
setPrivateField(source, "dataRegionId", 2147483646);
setPrivateField(source, "hasBeenStarted", true);
setPrivateField(source, "isDbNameCoveredByPattern", true);
@@ -216,7 +282,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
source,
"pendingQueue",
new ArrayDeque<TsFileResource>(Collections.singletonList(resource)));
- manager.pinTsFileResource(resource, false, pipeName);
+ manager.pinTsFileResource(resource, false, resourcePipeName);
event = (PipeTsFileInsertionEvent) source.supply();
Assert.assertNotNull(event);
@@ -237,6 +303,7 @@ public class PipeHistoricalDataRegionTsFileSourceTest {
source.close();
manager.cleanPipeTsFileDir(pipeName);
manager.cleanPipeTsFileDir(resourcePipeName);
+
FileUtils.deleteFileOrDirectory(tempDir);
}
}