This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a6676d3d529 Fix insertion cross space compaction recover null pointer
exception (#12439)
a6676d3d529 is described below
commit a6676d3d529cbe8d93c4069587d42e847398ef36
Author: shuwenwei <[email protected]>
AuthorDate: Tue May 7 20:00:52 2024 +0800
Fix insertion cross space compaction recover null pointer exception (#12439)
* fix insertion cross space compaction null pointer
* add ut
---
.../task/InsertionCrossSpaceCompactionTask.java | 13 +++-
.../InsertionCrossSpaceCompactionRecoverTest.java | 82 ++++++++++++++++++++++
2 files changed, 94 insertions(+), 1 deletion(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
index 6f262a79af6..d813bd066b3 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java
@@ -85,6 +85,8 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
super(databaseName, dataRegionId, 0L, tsFileManager, 0L,
CompactionTaskPriorityType.NORMAL);
this.logFile = logFile;
this.needRecoverTaskInfoFromLogFile = true;
+ this.selectedSeqFiles = Collections.emptyList();
+ this.selectedUnseqFiles = new ArrayList<>(1);
}
private TsFileResource unseqFileToInsert;
@@ -242,6 +244,7 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
File sourceTsFile =
sourceFileIdentifiers.get(0).getFileFromDataDirsIfAnyAdjuvantFileExists();
if (sourceTsFile != null) {
unseqFileToInsert = new TsFileResource(sourceTsFile);
+ selectedUnseqFiles.add(unseqFileToInsert);
}
File targetTsFile =
targetFileIdentifiers.get(0).getFileFromDataDirsIfAnyAdjuvantFileExists();
if (targetTsFile != null) {
@@ -288,7 +291,9 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
return targetFile == null
|| !targetFile.tsFileExists()
|| !targetFile.resourceFileExists()
- || (unseqFileToInsert.modFileExists() && !targetFile.modFileExists());
+ || (unseqFileToInsert != null
+ && unseqFileToInsert.modFileExists()
+ && !targetFile.modFileExists());
}
private void rollback() throws IOException {
@@ -298,6 +303,9 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
Collections.singletonList(targetFile),
Collections.singletonList(unseqFileToInsert));
}
deleteCompactionModsFile(Collections.singletonList(unseqFileToInsert));
+ if (targetFile == null) {
+ return;
+ }
if (targetFile.tsFileExists()) {
FileMetrics.getInstance().deleteTsFile(true,
Collections.singletonList(targetFile));
}
@@ -309,6 +317,9 @@ public class InsertionCrossSpaceCompactionTask extends
AbstractCompactionTask {
}
private void finishTask() throws IOException {
+ if (unseqFileToInsert == null) {
+ return;
+ }
if (recoverMemoryStatus && unseqFileToInsert.tsFileExists()) {
FileMetrics.getInstance().deleteTsFile(false,
Collections.singletonList(unseqFileToInsert));
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
index e99d2c7221c..69b4bc94fc0 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/InsertionCrossSpaceCompactionRecoverTest.java
@@ -345,6 +345,88 @@ public class InsertionCrossSpaceCompactionRecoverTest
extends AbstractCompaction
Assert.assertTrue(targetFile.modFileExists());
}
+ @Test
+ public void testRecoverWithTargetFileNotExist()
+ throws IllegalPathException, IOException, MergeException {
+ IDeviceID d1 = new PlainDeviceID("root.testsg.d1");
+ IDeviceID d2 = new PlainDeviceID("root.testsg.d2");
+
+ TsFileResource seqResource1 = createTsFileResource("1-1-0-0.tsfile", true);
+ seqResource1.updateStartTime(d1, 10);
+ seqResource1.updateEndTime(d1, 20);
+ seqResource1.updateStartTime(d2, 20);
+ seqResource1.updateEndTime(d2, 30);
+ createTsFileByResource(seqResource1);
+ seqResource1.serialize();
+
+ TsFileResource seqResource2 = createTsFileResource("3-3-0-0.tsfile", true);
+ seqResource2.updateStartTime(d1, 30);
+ seqResource2.updateEndTime(d1, 40);
+ seqResource2.updateStartTime(d2, 40);
+ seqResource2.updateEndTime(d2, 50);
+ createTsFileByResource(seqResource2);
+ seqResource2.serialize();
+
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+
+ TsFileResource unseqResource1 = createTsFileResource("9-9-0-0.tsfile",
false);
+ unseqResource1.updateStartTime(d1, 22);
+ unseqResource1.updateEndTime(d1, 25);
+ unseqResource1.updateStartTime(d2, 31);
+ unseqResource1.updateEndTime(d2, 37);
+ createTsFileByResource(unseqResource1);
+ unseqResource1.serialize();
+
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ deleteMap.put(((PlainDeviceID) d1).toStringID() + ".s1", new Pair<>(0L,
300L));
+ CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResource1,
false);
+
+ unseqResources.add(unseqResource1);
+
+ tsFileManager.addAll(seqResources, true);
+ tsFileManager.addAll(unseqResources, false);
+
+ RewriteCrossSpaceCompactionSelector selector =
+ new RewriteCrossSpaceCompactionSelector("root.testsg", "0", 0,
tsFileManager);
+ InsertionCrossCompactionTaskResource taskResource =
+ selector.selectOneInsertionTask(
+ new CrossSpaceCompactionCandidate(seqResources, unseqResources));
+ Assert.assertEquals(unseqResource1, taskResource.toInsertUnSeqFile);
+ Assert.assertEquals(seqResource1, taskResource.prevSeqFile);
+ Assert.assertEquals(seqResource2, taskResource.nextSeqFile);
+ Assert.assertEquals(unseqResource1,
taskResource.firstUnSeqFileInParitition);
+
+ InsertionCrossSpaceCompactionTask task =
+ new InsertionCrossSpaceCompactionTask(new Phaser(), 0, tsFileManager,
taskResource, 0);
+ TsFileResource targetFile = new TsFileResource(task.generateTargetFile());
+ File logFile =
+ new File(
+ targetFile.getTsFilePath() +
CompactionLogger.INSERTION_COMPACTION_LOG_NAME_SUFFIX);
+
+ CompactionFileGeneratorUtils.generateMods(deleteMap, unseqResource1, true);
+
+ try (SimpleCompactionLogger logger = new SimpleCompactionLogger(logFile)) {
+ logger.logSourceFile(taskResource.toInsertUnSeqFile);
+ logger.logTargetFile(targetFile);
+ logger.force();
+ }
+
+ // recover compaction, all target file and compaction.mods file should be
deleted and source
+ // file should be existed
+ new InsertionCrossSpaceCompactionTask("root.testsg", "0", tsFileManager,
logFile).recover();
+
+ Assert.assertTrue(unseqResource1.getTsFile().exists());
+ Assert.assertTrue(
+ new File(unseqResource1.getTsFilePath() +
TsFileResource.RESOURCE_SUFFIX).exists());
+ Assert.assertTrue(unseqResource1.getModFile().exists());
+ Assert.assertFalse(unseqResource1.getCompactionModFile().exists());
+
+ Assert.assertFalse(targetFile.tsFileExists());
+ Assert.assertFalse(targetFile.resourceFileExists());
+ Assert.assertFalse(targetFile.modFileExists());
+ }
+
private TsFileResource createTsFileResource(String name, boolean seq) {
String filePath = (seq ? SEQ_DIRS : UNSEQ_DIRS) + File.separator + name;
TsFileResource resource = new TsFileResource();
