This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 85e71a43543 Fix update time index after compact aligned series (#11943)
85e71a43543 is described below
commit 85e71a43543805cade34498cd784782f53dee630
Author: shuwenwei <[email protected]>
AuthorDate: Mon Jan 22 12:22:34 2024 +0800
Fix update time index after compact aligned series (#11943)
---
.../impl/ReadChunkCompactionPerformer.java | 1 +
.../ReadChunkAlignedSeriesCompactionExecutor.java | 1 -
...nkCompactionPerformerWithAlignedSeriesTest.java | 73 ++++++++++++++++++++++
3 files changed, 74 insertions(+), 1 deletion(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
index 230df876866..870c35deca1 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/ReadChunkCompactionPerformer.java
@@ -139,6 +139,7 @@ public class ReadChunkCompactionPerformer implements
ISeqCompactionPerformer {
targetResource.updateEndTime(device, chunkMetadata.getEndTime());
}
}
+ writer.checkMetadataSizeAndMayFlush();
writer.endChunkGroup();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
index d77479d9be2..a70a9832419 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/ReadChunkAlignedSeriesCompactionExecutor.java
@@ -177,7 +177,6 @@ public class ReadChunkAlignedSeriesCompactionExecutor {
if (!chunkWriter.isEmpty()) {
flushCurrentChunkWriter();
}
- writer.checkMetadataSizeAndMayFlush();
}
private void compactWithAlignedChunk(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/NewReadChunkCompactionPerformerWithAlignedSeriesTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/NewReadChunkCompactionPerformerWithAlignedSeriesTest.java
index 3cd710ba953..02b54233502 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/NewReadChunkCompactionPerformerWithAlignedSeriesTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/NewReadChunkCompactionPerformerWithAlignedSeriesTest.java
@@ -45,6 +45,7 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -569,6 +570,78 @@ public class
NewReadChunkCompactionPerformerWithAlignedSeriesTest extends Abstra
CompactionCheckerUtils.readFiles(Collections.singletonList(targetResource)));
}
+ @Test
+ public void testFlushChunkMetadataToTempFile()
+ throws IOException, StorageEngineException, InterruptedException,
MetadataException {
+ List<String> devices = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ devices.add("d" + i);
+ }
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource1)) {
+ for (String device : devices) {
+ writer.startChunkGroup(device);
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s0", "s1", "s2"),
+ new TimeRange[] {new TimeRange(1000, 2000), new TimeRange(3000,
5000)},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ }
+ writer.endFile();
+ }
+ seqResources.add(seqResource1);
+
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource2)) {
+ for (String device : devices) {
+ writer.startChunkGroup(device);
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s0", "s1", "s2"),
+ new TimeRange[] {new TimeRange(6000, 7000), new TimeRange(8000,
9000)},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ }
+ writer.endFile();
+ }
+
+ seqResources.add(seqResource2);
+
+ TsFileResource seqResource3 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource3)) {
+ for (String device : devices) {
+ writer.startChunkGroup(device);
+ writer.generateSimpleAlignedSeriesToCurrentDevice(
+ Arrays.asList("s0", "s1", "s2"),
+ new TimeRange[] {new TimeRange(16000, 17000), new TimeRange(18000,
19000)},
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ }
+ writer.endFile();
+ }
+ seqResources.add(seqResource3);
+
+ tsFileManager.addAll(seqResources, true);
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(seqResources,
true);
+
+ ReadChunkCompactionPerformer performer = new
ReadChunkCompactionPerformer();
+ CompactionTaskSummary summary = new CompactionTaskSummary();
+ performer.setSummary(summary);
+ performer.setSourceFiles(seqResources);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.perform();
+ CompactionUtils.moveTargetFile(
+ Collections.singletonList(targetResource), true, COMPACTION_TEST_SG);
+ TsFileResourceUtils.validateTsFileDataCorrectness(targetResource);
+ Assert.assertEquals(
+ CompactionCheckerUtils.readFiles(seqResources),
+
CompactionCheckerUtils.readFiles(Collections.singletonList(targetResource)));
+ Assert.assertEquals(devices.size(),
targetResource.buildDeviceTimeIndex().getDevices().size());
+ }
+
private TsFileResource generateSingleAlignedSeriesFile(
String device,
List<String> measurement,
