This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9f04de9 [IOTDB-2723] Fix sequence inner space compaction lose data
9f04de9 is described below
commit 9f04de9965554acb7be16241333210ea5faeee0e
Author: Liu Xuxin <[email protected]>
AuthorDate: Tue Mar 15 21:42:04 2022 +0800
[IOTDB-2723] Fix sequence inner space compaction lose data
---
.../utils/SingleSeriesCompactionExecutor.java | 9 ++-
.../InnerSpaceCompactionUtilsNoAlignedTest.java | 83 +++++++++++++++++++++-
2 files changed, 90 insertions(+), 2 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
index 5f71c0f..e4c529d 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/utils/SingleSeriesCompactionExecutor.java
@@ -209,7 +209,14 @@ public class SingleSeriesCompactionExecutor {
}
private void writeCachedChunkIntoChunkWriter() throws IOException {
- cachedChunk.getData().flip();
+ if (cachedChunk.getData().position() != 0) {
+ // If the position of cache chunk data buffer is 0,
+ // it means that the cache chunk is the first chunk cached,
+ // and it hasn't merged with any chunk yet.
+ // If we flip it, both the position and limit in the buffer will be 0,
+ // which leads to the lost of data.
+ cachedChunk.getData().flip();
+ }
writeChunkIntoChunkWriter(cachedChunk);
cachedChunk = null;
cachedChunkMetadata = null;
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
index 78a8401..8251b0d 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionUtilsNoAlignedTest.java
@@ -758,7 +758,7 @@ public class InnerSpaceCompactionUtilsNoAlignedTest {
}
@Test
- public void testMixCompact() throws Exception {
+ public void testMixCompact1() throws Exception {
long testTargetChunkPointNum = 2000L;
long testChunkSizeLowerBound = 1024L;
long testChunkPointNumLowerBound = 100L;
@@ -834,4 +834,85 @@ public class InnerSpaceCompactionUtilsNoAlignedTest {
.setChunkPointNumLowerBoundInCompaction(originChunkPointNumLowerBound);
}
}
+
+ @Test
+ public void testMixCompact2() throws Exception {
+ long testTargetChunkPointNum = 2000L;
+ long testChunkSizeLowerBound = 1024L;
+ long testChunkPointNumLowerBound = 100L;
+ long originTargetChunkSize =
IoTDBDescriptor.getInstance().getConfig().getTargetChunkSize();
+ long originTargetChunkPointNum =
+ IoTDBDescriptor.getInstance().getConfig().getTargetChunkPointNum();
+ IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024 * 1024);
+
IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(testTargetChunkPointNum);
+ long originChunkSizeLowerBound =
+
IoTDBDescriptor.getInstance().getConfig().getChunkSizeLowerBoundInCompaction();
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkSizeLowerBoundInCompaction(testChunkSizeLowerBound);
+ long originChunkPointNumLowerBound =
+
IoTDBDescriptor.getInstance().getConfig().getChunkPointNumLowerBoundInCompaction();
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkPointNumLowerBoundInCompaction(testChunkPointNumLowerBound);
+ try {
+ List<TsFileResource> sourceFiles = new ArrayList();
+ int fileNum = 12;
+ long pointStep = 10L;
+ long[] points = new long[] {1960, 50, 1960, 50, 2100, 50, 1960, 2300,
2500, 1000, 500, 500};
+ for (int i = 0; i < fileNum; ++i) {
+ List<List<Long>> chunkPagePointsNum = new ArrayList<>();
+ List<Long> pagePointsNum = new ArrayList<>();
+ pagePointsNum.add(points[i]);
+ chunkPagePointsNum.add(pagePointsNum);
+ TsFileResource resource =
+ new TsFileResource(new File(SEQ_DIRS,
String.format("%d-%d-0-0.tsfile", i + 1, i + 1)));
+ sourceFiles.add(resource);
+ CompactionFileGeneratorUtils.writeTsFile(
+ fullPathSet, chunkPagePointsNum, i * 2500L, resource);
+ }
+
+ Map<PartialPath, List<TimeValuePair>> originData =
+ CompactionCheckerUtils.getDataByQuery(paths, schemaList,
sourceFiles, new ArrayList<>());
+ TsFileNameGenerator.TsFileName tsFileName =
+
TsFileNameGenerator.getTsFileName(sourceFiles.get(0).getTsFile().getName());
+ TsFileResource targetResource =
+ new TsFileResource(
+ new File(
+ SEQ_DIRS,
+ String.format(
+ "%d-%d-%d-%d.tsfile",
+ tsFileName.getTime(),
+ tsFileName.getVersion(),
+ tsFileName.getInnerCompactionCnt() + 1,
+ tsFileName.getCrossCompactionCnt())));
+ InnerSpaceCompactionUtils.compact(targetResource, sourceFiles);
+ Map<String, List<List<Long>>> chunkPagePointsNumMerged = new HashMap<>();
+ // outer list is a chunk, inner list is point num in each page
+ for (String path : fullPathSet) {
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path,
2010);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path,
2010);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path,
2100);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path,
2010);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path,
2300);
+ CompactionCheckerUtils.putOnePageChunk(chunkPagePointsNumMerged, path,
2500);
+ CompactionCheckerUtils.putChunk(
+ chunkPagePointsNumMerged, path, new long[] {1000, 500, 500});
+ }
+ Map<PartialPath, List<TimeValuePair>> compactedData =
+ CompactionCheckerUtils.getDataByQuery(
+ paths, schemaList, Collections.singletonList(targetResource),
new ArrayList<>());
+ CompactionCheckerUtils.validDataByValueList(originData, compactedData);
+ CompactionCheckerUtils.checkChunkAndPage(chunkPagePointsNumMerged,
targetResource);
+ } finally {
+
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(originTargetChunkSize);
+
IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(originTargetChunkPointNum);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setChunkSizeLowerBoundInCompaction(originChunkSizeLowerBound);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+
.setChunkPointNumLowerBoundInCompaction(originChunkPointNumLowerBound);
+ }
+ }
}
