This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 238f230 [IOTDB-2581] Fix compaction recover in 0.13.0 is not
compatible with 0.12.5 (#5099)
238f230 is described below
commit 238f230a83d604d179deb36a176deee80c98374f
Author: Liu Xuxin <[email protected]>
AuthorDate: Fri Feb 25 20:54:13 2022 +0800
[IOTDB-2581] Fix compaction recover in 0.13.0 is not compatible with 0.12.5
(#5099)
---
.../db/engine/compaction/TsFileIdentifier.java | 12 +-
.../inner/utils/SizeTieredCompactionLogger.java | 4 +-
.../SizeTieredCompactionRecoverCompatibleTest.java | 157 +++++++++++++++++++++
3 files changed, 166 insertions(+), 7 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java
index daf8808..a93c34e 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileIdentifier.java
@@ -36,16 +36,18 @@ public class TsFileIdentifier {
private final boolean sequence;
private final String filename;
public static final String INFO_SEPARATOR = " ";
+ // Notice: Do not change the offset of info
public static final int FILE_NAME_OFFSET_IN_PATH = 1;
public static final int TIME_PARTITION_OFFSET_IN_PATH = 2;
public static final int VIRTUAL_SG_OFFSET_IN_PATH = 3;
public static final int LOGICAL_SG_OFFSET_IN_PATH = 4;
public static final int SEQUENCE_OFFSET_IN_PATH = 5;
+
public static final int LOGICAL_SG_OFFSET_IN_LOG = 0;
public static final int VIRTUAL_SG_OFFSET_IN_LOG = 1;
public static final int TIME_PARTITION_OFFSET_IN_LOG = 2;
- public static final int SEQUENCE_OFFSET_IN_LOG = 3;
- public static final int FILE_NAME_OFFSET_IN_LOG = 4;
+ public static final int FILE_NAME_OFFSET_IN_LOG = 3;
+ public static final int SEQUENCE_OFFSET_IN_LOG = 4;
private TsFileIdentifier(
String logicalStorageGroupName,
@@ -105,7 +107,7 @@ public class TsFileIdentifier {
splittedFileInfo[LOGICAL_SG_OFFSET_IN_LOG],
splittedFileInfo[VIRTUAL_SG_OFFSET_IN_LOG],
splittedFileInfo[TIME_PARTITION_OFFSET_IN_LOG],
- Boolean.parseBoolean(splittedFileInfo[SEQUENCE_OFFSET_IN_LOG]),
+ splittedFileInfo[SEQUENCE_OFFSET_IN_LOG].equals("sequence"),
splittedFileInfo[FILE_NAME_OFFSET_IN_LOG]);
}
@@ -119,9 +121,9 @@ public class TsFileIdentifier {
INFO_SEPARATOR,
timePartitionId,
INFO_SEPARATOR,
- sequence,
+ filename,
INFO_SEPARATOR,
- filename);
+ sequence ? "sequence" : "unsequence");
}
@Override
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SizeTieredCompactionLogger.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SizeTieredCompactionLogger.java
index 687449f..bc8d36b 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SizeTieredCompactionLogger.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SizeTieredCompactionLogger.java
@@ -32,8 +32,8 @@ public class SizeTieredCompactionLogger {
public static final String COMPACTION_LOG_NAME = ".compaction.log";
public static final String SOURCE_NAME = "source";
public static final String TARGET_NAME = "target";
- public static final String SOURCE_INFO = "source_info";
- public static final String TARGET_INFO = "target_info";
+ public static final String SOURCE_INFO = "info-source";
+ public static final String TARGET_INFO = "info-target";
public static final String SEQUENCE_NAME = "sequence";
public static final String UNSEQUENCE_NAME = "unsequence";
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
new file mode 100644
index 0000000..e9bd9f8
--- /dev/null
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/recover/SizeTieredCompactionRecoverCompatibleTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.engine.compaction.recover;
+
+import org.apache.iotdb.db.engine.compaction.AbstractCompactionTest;
+import
org.apache.iotdb.db.engine.compaction.inner.sizetiered.SizeTieredCompactionRecoverTask;
+import
org.apache.iotdb.db.engine.compaction.inner.utils.InnerSpaceCompactionUtils;
+import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** This test is used to test the compatibility of compaction recovery with
0.12. */
+public class SizeTieredCompactionRecoverCompatibleTest extends
AbstractCompactionTest {
+ @Override
+ @Before
+ public void setUp() throws IOException, MetadataException,
WriteProcessException {
+ super.setUp();
+ }
+
+ @Override
+ @After
+ public void tearDown() throws StorageEngineException, IOException {
+ super.tearDown();
+ }
+
+ @Test
+ public void testCompatibleWithFilePath() throws Exception {
+ createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
+ registerTimeseriesInMManger(2, 3, false);
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources,
true);
+ InnerSpaceCompactionUtils.compact(targetResource, seqResources);
+ RandomAccessFile targetFile = new
RandomAccessFile(targetResource.getTsFile(), "rw");
+ long fileLength = targetFile.length();
+ targetFile.getChannel().truncate(fileLength - 20);
+ targetFile.close();
+
+ File logFile =
+ new File(
+ targetResource.getTsFile().getParent(),
+ targetResource.getTsFile().getName() + ".compaction.log");
+ BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
+ for (TsFileResource tsFileResource : seqResources) {
+ logWriter.write(
+ String.format(
+ "info-source\nroot.compactionTest 0 0 %s sequence\n",
+ tsFileResource.getTsFile().getName()));
+ }
+ logWriter.write("sequence\n");
+ logWriter.write(
+ String.format(
+ "info-target\nroot.compactionTest 0 0 %s sequence\n",
+ targetResource.getTsFile().getName()));
+ logWriter.close();
+
+ TsFileManager tsFileManager =
+ new TsFileManager("root.compactionTest", "0",
targetResource.getTsFile().getParent());
+ tsFileManager.addAll(seqResources, true);
+ SizeTieredCompactionRecoverTask recoverTask =
+ new SizeTieredCompactionRecoverTask(
+ "root.compactionTest",
+ "0",
+ 0,
+ logFile,
+ targetResource.getTsFile().getParent(),
+ true,
+ new AtomicInteger(0),
+ tsFileManager);
+ recoverTask.call();
+
+ for (TsFileResource resource : seqResources) {
+ Assert.assertTrue(resource.getTsFile().exists());
+ }
+
+ Assert.assertFalse(targetResource.getTsFile().exists());
+ Assert.assertFalse(logFile.exists());
+ }
+
+ @Test
+ public void testCompatibleWithFileInfo() throws Exception {
+ createFiles(6, 2, 3, 100, 0, 0, 50, 50, false, true);
+ registerTimeseriesInMManger(2, 3, false);
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources,
true);
+ InnerSpaceCompactionUtils.compact(targetResource, seqResources);
+ RandomAccessFile targetFile = new
RandomAccessFile(targetResource.getTsFile(), "rw");
+ long fileLength = targetFile.length();
+ targetFile.getChannel().truncate(fileLength - 20);
+ targetFile.close();
+
+ File logFile =
+ new File(
+ targetResource.getTsFile().getParent(),
+ targetResource.getTsFile().getName() + ".compaction.log");
+ BufferedWriter logWriter = new BufferedWriter(new FileWriter(logFile));
+ for (TsFileResource tsFileResource : seqResources) {
+ logWriter.write(String.format("source\n%s\n",
tsFileResource.getTsFile().getAbsolutePath()));
+ }
+ logWriter.write("sequence\n");
+ logWriter.write(String.format("target\n%s\n",
targetResource.getTsFile().getAbsolutePath()));
+ logWriter.close();
+
+ TsFileManager tsFileManager =
+ new TsFileManager("root.compactionTest", "0",
targetResource.getTsFile().getParent());
+ tsFileManager.addAll(seqResources, true);
+ SizeTieredCompactionRecoverTask recoverTask =
+ new SizeTieredCompactionRecoverTask(
+ "root.compactionTest",
+ "0",
+ 0,
+ logFile,
+ targetResource.getTsFile().getParent(),
+ true,
+ new AtomicInteger(0),
+ tsFileManager);
+ recoverTask.call();
+
+ for (TsFileResource resource : seqResources) {
+ Assert.assertTrue(resource.getTsFile().exists());
+ }
+
+ Assert.assertFalse(targetResource.getTsFile().exists());
+ Assert.assertFalse(logFile.exists());
+ }
+}
