This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 196d6a9 [IOTDB-2712]Remove reading merge.mods in inner compaction
recover (#5186)
196d6a9 is described below
commit 196d6a9da2adcca8b912d91125f6f4eb94850b44
Author: 周沛辰 <[email protected]>
AuthorDate: Thu Mar 10 15:36:14 2022 +0800
[IOTDB-2712]Remove reading merge.mods in inner compaction recover (#5186)
---
.../SizeTieredCompactionRecoverTask.java | 26 +---
.../db/engine/compaction/TsFileIdentifierUT.java | 6 +-
.../SizeTieredCompactionRecoverCompatibleTest.java | 147 ++++++++++++++++++++-
3 files changed, 148 insertions(+), 31 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTask.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTask.java
index 97ae5c1..af90f86 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTask.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTask.java
@@ -22,7 +22,6 @@ import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.CompactionUtils;
import org.apache.iotdb.db.engine.compaction.TsFileIdentifier;
-import
org.apache.iotdb.db.engine.compaction.cross.rewrite.task.RewriteCrossCompactionRecoverTask;
import
org.apache.iotdb.db.engine.compaction.inner.InnerSpaceCompactionExceptionHandler;
import
org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils;
import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask;
@@ -173,9 +172,6 @@ public class SizeTieredCompactionRecoverTask extends
SizeTieredCompactionTask {
true);
} else {
handleSuccess =
handleWithoutAllSourceFilesExist(sourceFileIdentifiers);
- if (logAnalyzer.isLogFromOld()) {
- appendCompactionModificationsFromOld(targetResource);
- }
}
}
} catch (IOException e) {
@@ -270,7 +266,7 @@ public class SizeTieredCompactionRecoverTask extends
SizeTieredCompactionTask {
handleSuccess = false;
}
}
- // delete remaining source files
+ // delete remaining source files and resource files
if (!InnerSpaceCompactionUtils.deleteTsFilesInDisk(
remainSourceTsFileResources, fullStorageGroupName)) {
LOGGER.error(
@@ -297,26 +293,6 @@ public class SizeTieredCompactionRecoverTask extends
SizeTieredCompactionTask {
return null;
}
- /**
- * Append compaction modifications from previous version (<0.13) to target
mods file.
- *
- * @param targetResource
- */
- private void appendCompactionModificationsFromOld(TsFileResource
targetResource)
- throws IOException {
- File compactionModsFileFromOld =
- new File(
- tsFileManager.getStorageGroupDir()
- + File.separator
- + IoTDBConstant.COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD);
- if (compactionModsFileFromOld.exists()) {
- ModificationFile compactionModsFile =
- new ModificationFile(compactionModsFileFromOld.getPath());
- RewriteCrossCompactionRecoverTask.appendCompactionModificationsFromOld(
- targetResource, compactionModsFile);
- }
- }
-
/** Return whether compaction log file is from previous version (<0.13). */
private boolean isOldLog() {
return
compactionLogFile.getName().startsWith(tsFileManager.getStorageGroupName());
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
index d2078b3..5824bf5 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifierUT.java
@@ -90,7 +90,7 @@ public class TsFileIdentifierUT {
@Test
public void testGetInfoFromInfoString() {
- String[] firstInfoArray = new String[] {"root.test.sg", "0", "0", "true",
"1-1-0-0.tsfile"};
+ String[] firstInfoArray = new String[] {"sequence", "root.test.sg", "0",
"0", "1-1-0-0.tsfile"};
String firstInfoString = String.join(INFO_SEPARATOR, firstInfoArray);
TsFileIdentifier firstInfo =
TsFileIdentifier.getFileIdentifierFromInfoString(firstInfoString);
Assert.assertEquals(firstInfo.getFilename(), "1-1-0-0.tsfile");
@@ -100,7 +100,7 @@ public class TsFileIdentifierUT {
Assert.assertTrue(firstInfo.isSequence());
String[] secondInfoArray =
- new String[] {"root.test.sg", "0", "425", "false",
"666-888-222-131.tsfile"};
+ new String[] {"unsequence", "root.test.sg", "0", "425",
"666-888-222-131.tsfile"};
String secondInfoString = String.join(INFO_SEPARATOR, secondInfoArray);
TsFileIdentifier secondInfo =
TsFileIdentifier.getFileIdentifierFromInfoString(secondInfoString);
@@ -110,7 +110,7 @@ public class TsFileIdentifierUT {
Assert.assertEquals(secondInfo.getLogicalStorageGroupName(),
"root.test.sg");
Assert.assertFalse(secondInfo.isSequence());
- String[] illegalInfoArray = new String[] {"0", "425", "false",
"666-888-222-131.tsfile"};
+ String[] illegalInfoArray = new String[] {"unsequence", "0", "425",
"666-888-222-131.tsfile"};
String illegalInfoString = String.join(INFO_SEPARATOR, illegalInfoArray);
try {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
index e666d37..4d59def 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
@@ -22,12 +22,15 @@ package org.apache.iotdb.db.engine.compaction.recover;
import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
import
org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionRecoverTask;
import
org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils;
+import
org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils;
+import org.apache.iotdb.db.engine.compaction.utils.log.CompactionLogger;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.utils.Pair;
import org.junit.After;
import org.junit.Assert;
@@ -39,8 +42,12 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.RandomAccessFile;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
+import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR;
+
/** This test is used to test the compatibility of compaction recovery with
0.12. */
public class SizeTieredCompactionRecoverCompatibleTest extends
AbstractCompactionTest {
@Override
@@ -56,7 +63,7 @@ public class SizeTieredCompactionRecoverCompatibleTest
extends AbstractCompactio
}
@Test
- public void testCompatibleWithFilePath() throws Exception {
+ public void testCompatibleWithAllSourceFilesExistWithFilePath() throws
Exception {
createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
registerTimeseriesInMManger(2, 3, false);
TsFileResource targetResource =
@@ -67,8 +74,18 @@ public class SizeTieredCompactionRecoverCompatibleTest
extends AbstractCompactio
targetFile.getChannel().truncate(fileLength - 20);
targetFile.close();
+ for (int i = 0; i < seqResources.size(); i++) {
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
+ new Pair(i * 10L, i * 10L + 10));
+ CompactionFileGeneratorUtils.generateMods(deleteMap,
seqResources.get(i), false);
+ }
+
File logFile =
- new File(targetResource.getTsFile().getParent(), "root.compactionTest"
+ ".compaction.log");
+ new File(
+ targetResource.getTsFile().getParent(),
+ "root.compactionTest" +
CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX_FROM_OLD);
BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
for (TsFileResource tsFileResource : seqResources) {
logWriter.write(
@@ -100,14 +117,82 @@ public class SizeTieredCompactionRecoverCompatibleTest
extends AbstractCompactio
for (TsFileResource resource : seqResources) {
Assert.assertTrue(resource.getTsFile().exists());
+ Assert.assertTrue(resource.resourceFileExists());
+ Assert.assertTrue(resource.getModFile().exists());
}
Assert.assertFalse(targetResource.getTsFile().exists());
+ Assert.assertFalse(targetResource.resourceFileExists());
+ Assert.assertFalse(targetResource.getModFile().exists());
Assert.assertFalse(logFile.exists());
}
@Test
- public void testCompatibleWithFileInfo() throws Exception {
+ public void testCompatibleWithSomeSourceFilesLostWithFilePath() throws
Exception {
+ createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
+ registerTimeseriesInMManger(2, 3, false);
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources,
true);
+ InnerSpaceCompactionUtils.compact(targetResource, seqResources);
+ InnerSpaceCompactionUtils.moveTargetFile(targetResource,
"root.compactionTest");
+
+ // first source file does not exist
+ seqResources.get(0).delete();
+
+ for (int i = 0; i < seqResources.size(); i++) {
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
+ new Pair(i * 10L, i * 10L + 10));
+ CompactionFileGeneratorUtils.generateMods(deleteMap,
seqResources.get(i), false);
+ }
+
+ File logFile =
+ new File(
+ targetResource.getTsFile().getParent(),
+ "root.compactionTest" +
CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX_FROM_OLD);
+ BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
+ for (TsFileResource tsFileResource : seqResources) {
+ logWriter.write(
+ String.format(
+ "info-source\nroot.compactionTest 0 0 %s sequence\n",
+ tsFileResource.getTsFile().getName()));
+ }
+ logWriter.write("sequence\n");
+ logWriter.write(
+ String.format(
+ "info-target\nroot.compactionTest 0 0 %s sequence\n",
+ targetResource.getTsFile().getName()));
+ logWriter.close();
+
+ TsFileManager tsFileManager =
+ new TsFileManager("root.compactionTest", "0",
targetResource.getTsFile().getParent());
+ tsFileManager.addAll(seqResources, true);
+ SizeTieredCompactionRecoverTask recoverTask =
+ new SizeTieredCompactionRecoverTask(
+ "root.compactionTest",
+ "0",
+ 0,
+ logFile,
+ targetResource.getTsFile().getParent(),
+ true,
+ new AtomicInteger(0),
+ tsFileManager);
+ recoverTask.call();
+
+ for (TsFileResource resource : seqResources) {
+ Assert.assertFalse(resource.getTsFile().exists());
+ Assert.assertFalse(resource.resourceFileExists());
+ Assert.assertFalse(resource.getModFile().exists());
+ }
+
+ Assert.assertTrue(targetResource.getTsFile().exists());
+ Assert.assertTrue(targetResource.resourceFileExists());
+ Assert.assertFalse(logFile.exists());
+ }
+
+ @Test
+ public void testCompatibleWithAllSourceFilesExistWithFileInfo() throws
Exception {
createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
registerTimeseriesInMManger(2, 3, false);
TsFileResource targetResource =
@@ -150,4 +235,60 @@ public class SizeTieredCompactionRecoverCompatibleTest
extends AbstractCompactio
Assert.assertFalse(targetResource.getTsFile().exists());
Assert.assertFalse(logFile.exists());
}
+
+ @Test
+ public void testCompatibleWithSomeSourceFilesLostWithFileInfo() throws
Exception {
+ createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, false);
+ registerTimeseriesInMManger(2, 3, false);
+ TsFileResource targetResource =
+
TsFileNameGenerator.getInnerCompactionTargetFileResource(unseqResources, true);
+ InnerSpaceCompactionUtils.compact(targetResource, unseqResources);
+ InnerSpaceCompactionUtils.moveTargetFile(targetResource,
"root.compactionTest");
+
+ // first source file does not exist
+ unseqResources.get(0).delete();
+
+ for (int i = 0; i < unseqResources.size(); i++) {
+ Map<String, Pair<Long, Long>> deleteMap = new HashMap<>();
+ deleteMap.put(
+ COMPACTION_TEST_SG + PATH_SEPARATOR + "d0" + PATH_SEPARATOR + "s0",
+ new Pair(i * 10L, i * 10L + 10));
+ CompactionFileGeneratorUtils.generateMods(deleteMap,
unseqResources.get(i), false);
+ }
+
+ File logFile =
+ new File(targetResource.getTsFile().getParent(), "root.compactionTest"
+ ".compaction.log");
+ BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
+ for (TsFileResource tsFileResource : unseqResources) {
+ logWriter.write(String.format("source\n%s\n",
tsFileResource.getTsFile().getAbsolutePath()));
+ }
+ logWriter.write("sequence\n");
+ logWriter.write(String.format("target\n%s\n",
targetResource.getTsFile().getAbsolutePath()));
+ logWriter.close();
+
+ TsFileManager tsFileManager =
+ new TsFileManager("root.compactionTest", "0",
targetResource.getTsFile().getParent());
+ tsFileManager.addAll(unseqResources, false);
+ SizeTieredCompactionRecoverTask recoverTask =
+ new SizeTieredCompactionRecoverTask(
+ "root.compactionTest",
+ "0",
+ 0,
+ logFile,
+ targetResource.getTsFile().getParent(),
+ true,
+ new AtomicInteger(0),
+ tsFileManager);
+ recoverTask.call();
+
+ for (TsFileResource resource : unseqResources) {
+ Assert.assertFalse(resource.getTsFile().exists());
+ Assert.assertFalse(resource.resourceFileExists());
+ Assert.assertFalse(resource.getModFile().exists());
+ }
+
+ Assert.assertTrue(targetResource.getTsFile().exists());
+ Assert.assertTrue(targetResource.resourceFileExists());
+ Assert.assertFalse(logFile.exists());
+ }
}
