This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new 8c10a2f399 [To rel/1.1][IOTDB-5859]Compaction error when using Version
as first sort dimension (#9818)
8c10a2f399 is described below
commit 8c10a2f399a05596b04164145f1456ab5fd995c0
Author: 周沛辰 <[email protected]>
AuthorDate: Wed May 10 22:40:48 2023 +0800
[To rel/1.1][IOTDB-5859]Compaction error when using Version as first sort
dimension (#9818)
---
.../db/engine/storagegroup/TsFileResource.java | 10 +-
.../db/engine/storagegroup/TsFileResourceList.java | 62 ++++++------
.../engine/compaction/AbstractCompactionTest.java | 28 +++---
.../ReadPointCompactionPerformerTest.java | 16 ++-
.../cross/CrossSpaceCompactionExceptionTest.java | 5 +-
.../RewriteCrossSpaceCompactionRecoverTest.java | 10 +-
...eCrossSpaceCompactionWithFastPerformerTest.java | 22 ++++-
...sSpaceCompactionWithReadPointPerformerTest.java | 21 +++-
.../SizeTieredCompactionRecoverTest.java | 52 ++++++----
.../storagegroup/TsFileResourceListTest.java | 110 +++++++++++++++++++++
10 files changed, 257 insertions(+), 79 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 5ab10f08b5..340d2b3dfa 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -1033,7 +1033,15 @@ public class TsFileResource {
TsFileNameGenerator.getTsFileName(o1.getTsFile().getName());
TsFileNameGenerator.TsFileName n2 =
TsFileNameGenerator.getTsFileName(o2.getTsFile().getName());
- return (int) (n2.getVersion() - n1.getVersion());
+ long timeDiff = n2.getTime() - n1.getTime();
+ if (timeDiff != 0) {
+ return timeDiff < 0 ? -1 : 1;
+ }
+ long versionDiff = n2.getVersion() - n1.getVersion();
+ if (versionDiff != 0) {
+ return versionDiff < 0 ? -1 : 1;
+ }
+ return 0;
} catch (IOException e) {
return 0;
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceList.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceList.java
index a7c5ea8b3f..4209d8531a 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceList.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceList.java
@@ -143,43 +143,43 @@ public class TsFileResourceList implements
List<TsFileResource> {
return false;
}
if (tail == null) {
+ // empty list
header = newNode;
tail = newNode;
count++;
} else {
- // find the position to insert of this node
- // the list should be ordered by file timestamp
- long versionOfNewNode =
-
TsFileNameGenerator.getTsFileName(newNode.getTsFile().getName()).getVersion();
-
- if
(TsFileNameGenerator.getTsFileName(header.getTsFile().getName()).getVersion()
- > versionOfNewNode) {
- // the timestamp of head node is greater than the new node
- // insert it before the head
- insertBefore(header, newNode);
- } else if
(TsFileNameGenerator.getTsFileName(tail.getTsFile().getName()).getVersion()
- < versionOfNewNode) {
- // the timestamp of new node is greater than the tail node
- // insert it after the tail
+ TsFileNameGenerator.TsFileName newTsFileName =
+ TsFileNameGenerator.getTsFileName(newNode.getTsFile().getName());
+ long timestampOfNewNode = newTsFileName.getTime();
+ long versionOfNewNode = newTsFileName.getVersion();
+
+ // find the position to insert, the list should be ordered by file
timestamp. If timestamp is
+ // equal to each other, then list should be ordered by file version.
+ boolean isNewNodeAtTail = true;
+ TsFileResource currNode = header;
+ while (currNode != null) {
+ TsFileNameGenerator.TsFileName currTsFileName =
+ TsFileNameGenerator.getTsFileName(currNode.getTsFile().getName());
+ if (timestampOfNewNode == currTsFileName.getTime()
+ && versionOfNewNode < currTsFileName.getVersion()) {
+ // the timestamp of new node is equal to the current node and the
version of new node is
+ // less than current node, then insert new node before the current
node
+ isNewNodeAtTail = false;
+ break;
+ } else if (timestampOfNewNode < currTsFileName.getTime()) {
+ // the timestamp of new node is less than the current node, insert
new node before then
+ // current node
+ isNewNodeAtTail = false;
+ break;
+ }
+ currNode = currNode.next;
+ }
+
+ // insert new node
+ if (isNewNodeAtTail) {
insertAfter(tail, newNode);
} else {
- // the timestamp of new node is between the timestamp of head and tail
node
- // find the first node whose timestamp is greater than new node
- // and insert the new node before this node
- TsFileResource currNode = header;
- while (currNode.next != null) {
- if
(TsFileNameGenerator.getTsFileName(currNode.getTsFile().getName()).getVersion()
- > versionOfNewNode) {
- break;
- }
- currNode = currNode.next;
- }
- if
(TsFileNameGenerator.getTsFileName(currNode.getTsFile().getName()).getVersion()
- < versionOfNewNode) {
- LOGGER.error("Cannot find an appropriate place to insert {}",
newNode);
- } else {
- insertBefore(currNode, newNode);
- }
+ insertBefore(currNode, newNode);
}
}
return true;
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
index 9d205ad2de..f4a7528173 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
@@ -87,6 +87,13 @@ public class AbstractCompactionTest {
protected int maxDeviceNum = 25;
protected int maxMeasurementNum = 25;
+ private long[] timestamp = {0, 0, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6,
6, 7, 7, 7};
+ // seq version is not in order
+ private int[] seqVersion = {18, 19, 0, 1, 14, 15, 16, 2, 3, 4, 12, 13, 5, 6,
9, 10, 11, 7, 8, 17};
+ private int[] unseqVersion = {
+ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
38, 39
+ };
+
private static final long oldTargetChunkSize =
IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
@@ -147,11 +154,14 @@ public class AbstractCompactionTest {
private int fileVersion = 0;
+ private int fileCount = 0;
+
protected TsFileManager tsFileManager =
new TsFileManager(COMPACTION_TEST_SG, "0", STORAGE_GROUP_DIR.getPath());
public void setUp()
throws IOException, WriteProcessException, MetadataException,
InterruptedException {
+ fileCount = 0;
if (!SEQ_DIRS.exists()) {
Assert.assertTrue(SEQ_DIRS.mkdirs());
}
@@ -196,11 +206,9 @@ public class AbstractCompactionTest {
boolean isSeq)
throws IOException, WriteProcessException, MetadataException {
for (int i = 0; i < fileNum; i++) {
+ long version = isSeq ? seqVersion[fileCount] : unseqVersion[fileCount];
String fileName =
- System.currentTimeMillis()
- + FilePathUtils.FILE_NAME_SEPARATOR
- + fileVersion++
- + "-0-0.tsfile";
+ timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + version
+ "-0-0.tsfile";
String filePath;
if (isSeq) {
filePath = SEQ_DIRS.getPath() + File.separator + fileName;
@@ -270,11 +278,9 @@ public class AbstractCompactionTest {
throws IOException, WriteProcessException {
String value = isSeq ? "seqTestValue" : "unseqTestValue";
for (int i = 0; i < fileNum; i++) {
+ long version = isSeq ? seqVersion[fileCount] : unseqVersion[fileCount];
String fileName =
- System.currentTimeMillis()
- + FilePathUtils.FILE_NAME_SEPARATOR
- + fileVersion++
- + "-0-0.tsfile";
+ timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + version
+ "-0-0.tsfile";
String filePath;
if (isSeq) {
filePath = SEQ_DIRS.getPath() + File.separator + fileName;
@@ -576,11 +582,9 @@ public class AbstractCompactionTest {
}
protected TsFileResource createEmptyFileAndResource(boolean isSeq) {
+ long version = isSeq ? seqVersion[fileCount] : unseqVersion[fileCount];
String fileName =
- System.currentTimeMillis()
- + FilePathUtils.FILE_NAME_SEPARATOR
- + fileVersion
- + "-0-0.tsfile";
+ timestamp[fileCount++] + FilePathUtils.FILE_NAME_SEPARATOR + version +
"-0-0.tsfile";
String filePath;
if (isSeq) {
filePath = SEQ_DIRS.getPath() + File.separator + fileName;
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java
index a50533d809..d1441d3f2d 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/ReadPointCompactionPerformerTest.java
@@ -1718,7 +1718,12 @@ public class ReadPointCompactionPerformerTest extends
AbstractCompactionTest {
schemas);
IDataBlockReader tsBlockReader =
new SeriesDataBlockReader(
- path, EnvironmentUtils.TEST_QUERY_FI_CONTEXT, seqResources,
unseqResources, true);
+ path,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
+ seqResources,
+ unseqResources,
+ true);
int count = 0;
while (tsBlockReader.hasNextBatch()) {
TsBlock block = tsBlockReader.nextBatch();
@@ -4243,7 +4248,8 @@ public class ReadPointCompactionPerformerTest extends
AbstractCompactionTest {
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
targetResources,
new ArrayList<>(),
true);
@@ -4382,7 +4388,8 @@ public class ReadPointCompactionPerformerTest extends
AbstractCompactionTest {
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
targetResources,
new ArrayList<>(),
true);
@@ -5463,7 +5470,8 @@ public class ReadPointCompactionPerformerTest extends
AbstractCompactionTest {
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
targetResources,
new ArrayList<>(),
true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
index 6bbaa0a49b..06dfac3750 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionExceptionTest.java
@@ -619,8 +619,9 @@ public class CrossSpaceCompactionExceptionTest extends
AbstractCompactionTest {
Assert.assertFalse(resource.getCompactionModFile().exists());
}
// the first target file should be deleted after compaction, the others
still exist
- for (TsFileResource resource : targetResources) {
- if (resource.getVersion() == 0) {
+ for (int i = 0; i < targetResources.size(); i++) {
+ TsFileResource resource = targetResources.get(i);
+ if (i == 0) {
Assert.assertFalse(resource.getTsFile().exists());
Assert.assertFalse(resource.resourceFileExists());
} else {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java
index b54b57d7dd..8310dc56f8 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionRecoverTest.java
@@ -725,8 +725,9 @@ public class RewriteCrossSpaceCompactionRecoverTest extends
AbstractCompactionTe
Assert.assertFalse(resource.getCompactionModFile().exists());
}
// the first target file should be deleted after recovery
- for (TsFileResource resource : targetResources) {
- if (resource.getVersion() == 0) {
+ for (int i = 0; i < targetResources.size(); i++) {
+ TsFileResource resource = targetResources.get(i);
+ if (i == 0) {
Assert.assertFalse(resource.getTsFile().exists());
Assert.assertFalse(resource.resourceFileExists());
} else {
@@ -808,8 +809,9 @@ public class RewriteCrossSpaceCompactionRecoverTest extends
AbstractCompactionTe
Assert.assertFalse(resource.getCompactionModFile().exists());
}
// the first target file should be deleted after recovery
- for (TsFileResource resource : targetResources) {
- if (resource.getVersion() == 0) {
+ for (int i = 0; i < targetResources.size(); i++) {
+ TsFileResource resource = targetResources.get(i);
+ if (i == 0) {
Assert.assertFalse(resource.getTsFile().exists());
Assert.assertFalse(resource.resourceFileExists());
} else {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java
index 5528caaaeb..7c8fd2c19b 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithFastPerformerTest.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.wal.recover.WALRecoverManager;
@@ -152,6 +153,7 @@ public class
RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo
i < TsFileGeneratorUtils.getAlignDeviceOffset() + 4;
i++) {
for (int j = 0; j < 5; j++) {
+ System.out.println(i + "," + j);
List<IMeasurementSchema> schemas = new ArrayList<>();
schemas.add(new MeasurementSchema("s" + j, TSDataType.INT64));
AlignedPath path =
@@ -161,7 +163,12 @@ public class
RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo
schemas);
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
- path, EnvironmentUtils.TEST_QUERY_FI_CONTEXT, seqResources,
unseqResources, true);
+ path,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
+ seqResources,
+ unseqResources,
+ true);
int count = 0;
while (tsFilesReader.hasNextBatch()) {
TsBlock batchData = tsFilesReader.nextBatch();
@@ -258,7 +265,8 @@ public class
RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -383,7 +391,12 @@ public class
RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo
schemas);
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
- path, EnvironmentUtils.TEST_QUERY_FI_CONTEXT, seqResources,
unseqResources, true);
+ path,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
+ seqResources,
+ unseqResources,
+ true);
int count = 0;
while (tsFilesReader.hasNextBatch()) {
TsBlock batchData = tsFilesReader.nextBatch();
@@ -481,7 +494,8 @@ public class
RewriteCrossSpaceCompactionWithFastPerformerTest extends AbstractCo
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java
index 088dff33df..9a3cc04bb5 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionWithReadPointPerformerTest.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.wal.recover.WALRecoverManager;
@@ -161,7 +162,12 @@ public class
RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr
schemas);
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
- path, EnvironmentUtils.TEST_QUERY_FI_CONTEXT, seqResources,
unseqResources, true);
+ path,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
+ seqResources,
+ unseqResources,
+ true);
int count = 0;
while (tsFilesReader.hasNextBatch()) {
TsBlock batchData = tsFilesReader.nextBatch();
@@ -258,7 +264,8 @@ public class
RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -383,7 +390,12 @@ public class
RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr
schemas);
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
- path, EnvironmentUtils.TEST_QUERY_FI_CONTEXT, seqResources,
unseqResources, true);
+ path,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
+ seqResources,
+ unseqResources,
+ true);
int count = 0;
while (tsFilesReader.hasNextBatch()) {
TsBlock batchData = tsFilesReader.nextBatch();
@@ -481,7 +493,8 @@ public class
RewriteCrossSpaceCompactionWithReadPointPerformerTest extends Abstr
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+
FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java
index 5f12ae5a0d..859b21abe7 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionRecoverTest.java
@@ -41,6 +41,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.db.utils.SchemaTestUtils;
@@ -105,7 +106,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -191,7 +193,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -222,7 +225,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -297,7 +301,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -328,7 +333,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -405,7 +411,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -436,7 +443,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -508,7 +516,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true).subList(3, 6),
new ArrayList<>(),
true);
@@ -757,7 +766,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -820,7 +830,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true).subList(0, 5),
new ArrayList<>(),
true);
@@ -851,7 +862,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -907,7 +919,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -939,7 +952,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -997,7 +1011,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -1039,7 +1054,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -1079,7 +1095,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
@@ -1134,7 +1151,8 @@ public class SizeTieredCompactionRecoverTest extends
AbstractInnerSpaceCompactio
IDataBlockReader tsFilesReader =
new SeriesDataBlockReader(
path,
- EnvironmentUtils.TEST_QUERY_FI_CONTEXT,
+ FragmentInstanceContext.createFragmentInstanceContextForCompaction(
+ EnvironmentUtils.TEST_QUERY_CONTEXT.getQueryId()),
tsFileManager.getTsFileList(true),
new ArrayList<>(),
true);
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceListTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceListTest.java
index 15edccca8e..7c447eb904 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceListTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/TsFileResourceListTest.java
@@ -25,6 +25,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.io.File;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -212,4 +213,113 @@ public class TsFileResourceListTest {
tsFileResourceList.getHeader(),
tsFileResources.get(tsFileResources.size() - 1));
Assert.assertEquals(tsFileResourceList.getTail(), tsFileResources.get(0));
}
+
+ /** Seq files: 1-1, 1-5, 1-6, 2-4, 3-3, 4-7, 4-9, 5-8 */
+ @Test
+ public void testKeepOrderInsertWithSameTimestampAndDifferentVersion() throws
IOException {
+ List<TsFileResource> seqResources = new ArrayList<>();
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 1, 1, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 1, 5, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 1, 6, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 2, 4, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 3, 3, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 4, 7, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 4, 9, 0, 0))));
+ seqResources.add(
+ new TsFileResource(
+ new File(
+ TsFileNameGenerator.generateNewTsFilePath(
+ TestConstant.BASE_OUTPUT_PATH, 5, 8, 0, 0))));
+
+ TsFileResourceList tsFileResourceList = new TsFileResourceList();
+ tsFileResourceList.keepOrderInsert(seqResources.get(4));
+ tsFileResourceList.keepOrderInsert(seqResources.get(3));
+ tsFileResourceList.keepOrderInsert(seqResources.get(5));
+ tsFileResourceList.keepOrderInsert(seqResources.get(1));
+ tsFileResourceList.keepOrderInsert(seqResources.get(0));
+ tsFileResourceList.keepOrderInsert(seqResources.get(2));
+ tsFileResourceList.keepOrderInsert(seqResources.get(6));
+ tsFileResourceList.keepOrderInsert(seqResources.get(7));
+ Assert.assertEquals(seqResources, tsFileResourceList.getArrayList());
+
+ for (TsFileResource resource : seqResources) {
+ tsFileResourceList.remove(resource);
+ }
+ tsFileResourceList.keepOrderInsert(seqResources.get(0));
+ tsFileResourceList.keepOrderInsert(seqResources.get(4));
+ tsFileResourceList.keepOrderInsert(seqResources.get(5));
+ tsFileResourceList.keepOrderInsert(seqResources.get(7));
+ tsFileResourceList.keepOrderInsert(seqResources.get(2));
+ tsFileResourceList.keepOrderInsert(seqResources.get(1));
+ tsFileResourceList.keepOrderInsert(seqResources.get(3));
+ tsFileResourceList.keepOrderInsert(seqResources.get(6));
+ Assert.assertEquals(seqResources, tsFileResourceList.getArrayList());
+
+ for (TsFileResource resource : seqResources) {
+ tsFileResourceList.remove(resource);
+ }
+ tsFileResourceList.keepOrderInsert(seqResources.get(7));
+ tsFileResourceList.keepOrderInsert(seqResources.get(1));
+ tsFileResourceList.keepOrderInsert(seqResources.get(5));
+ tsFileResourceList.keepOrderInsert(seqResources.get(6));
+ tsFileResourceList.keepOrderInsert(seqResources.get(2));
+ tsFileResourceList.keepOrderInsert(seqResources.get(4));
+ tsFileResourceList.keepOrderInsert(seqResources.get(3));
+ tsFileResourceList.keepOrderInsert(seqResources.get(0));
+ Assert.assertEquals(seqResources, tsFileResourceList.getArrayList());
+
+ for (TsFileResource resource : seqResources) {
+ tsFileResourceList.remove(resource);
+ }
+ tsFileResourceList.keepOrderInsert(seqResources.get(0));
+ tsFileResourceList.keepOrderInsert(seqResources.get(1));
+ tsFileResourceList.keepOrderInsert(seqResources.get(2));
+ tsFileResourceList.keepOrderInsert(seqResources.get(3));
+ tsFileResourceList.keepOrderInsert(seqResources.get(4));
+ tsFileResourceList.keepOrderInsert(seqResources.get(5));
+ tsFileResourceList.keepOrderInsert(seqResources.get(6));
+ tsFileResourceList.keepOrderInsert(seqResources.get(7));
+ Assert.assertEquals(seqResources, tsFileResourceList.getArrayList());
+
+ for (TsFileResource resource : seqResources) {
+ tsFileResourceList.remove(resource);
+ }
+ tsFileResourceList.keepOrderInsert(seqResources.get(7));
+ tsFileResourceList.keepOrderInsert(seqResources.get(6));
+ tsFileResourceList.keepOrderInsert(seqResources.get(5));
+ tsFileResourceList.keepOrderInsert(seqResources.get(4));
+ tsFileResourceList.keepOrderInsert(seqResources.get(3));
+ tsFileResourceList.keepOrderInsert(seqResources.get(2));
+ tsFileResourceList.keepOrderInsert(seqResources.get(1));
+ tsFileResourceList.keepOrderInsert(seqResources.get(0));
+ Assert.assertEquals(seqResources, tsFileResourceList.getArrayList());
+ }
}
