This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new a7168a7173 [To rel/1.1][IOTDB-5897] Fix NullPointerException in
compaction (#9886)
a7168a7173 is described below
commit a7168a71736cc703e8e16a63854f6f18801b4390
Author: Liu Xuxin <[email protected]>
AuthorDate: Thu May 18 18:35:10 2023 +0800
[To rel/1.1][IOTDB-5897] Fix NullPointerException in compaction (#9886)
---
.../execute/utils/MultiTsFileDeviceIterator.java | 3 +-
.../ReadChunkCompactionPerformerAlignedTest.java | 76 ++++++++++++++++++++++
.../compaction/utils/CompactionCheckerUtils.java | 4 ++
3 files changed, 82 insertions(+), 1 deletion(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 388cb48f0f..7e2484e0d7 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -362,7 +362,8 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
if (modification.getDevice().equals(currentDevice.left)) {
for (int i = 0; i < valueChunkMetadataList.size(); ++i) {
IChunkMetadata chunkMetadata = valueChunkMetadataList.get(i);
- if
(modification.getMeasurement().equals(chunkMetadata.getMeasurementUid())) {
+ if (chunkMetadata != null
+ &&
modification.getMeasurement().equals(chunkMetadata.getMeasurementUid())) {
modificationForCurDevice.get(i).add(modification);
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
index 09997f8d7b..66cba77a90 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/inner/ReadChunkCompactionPerformerAlignedTest.java
@@ -32,14 +32,21 @@ import
org.apache.iotdb.db.engine.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.engine.compaction.utils.CompactionCheckerUtils;
import org.apache.iotdb.db.engine.compaction.utils.CompactionConfigRestorer;
import
org.apache.iotdb.db.engine.compaction.utils.CompactionFileGeneratorUtils;
+import org.apache.iotdb.db.engine.modification.Deletion;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TimeValuePair;
+import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.Pair;
+import org.apache.iotdb.tsfile.utils.TsPrimitiveType;
+import org.apache.iotdb.tsfile.write.chunk.AlignedChunkWriterImpl;
import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter;
+import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
import org.apache.commons.io.FileUtils;
import org.junit.After;
@@ -652,4 +659,73 @@ public class ReadChunkCompactionPerformerAlignedTest {
new ArrayList<>());
CompactionCheckerUtils.validDataByValueList(originData, compactedData);
}
+
+ @Test
+ public void testEmptyChunkWithModification() throws Exception {
+ List<IMeasurementSchema> schemas = new ArrayList<>();
+ schemas.add(new MeasurementSchema("s0", TSDataType.DOUBLE));
+ schemas.add(new MeasurementSchema("s1", TSDataType.FLOAT));
+ schemas.add(new MeasurementSchema("s2", TSDataType.INT64));
+ schemas.add(new MeasurementSchema("s3", TSDataType.INT32));
+ schemas.add(new MeasurementSchema("s4", TSDataType.TEXT));
+ schemas.add(new MeasurementSchema("s5", TSDataType.BOOLEAN));
+ Map<PartialPath, List<TimeValuePair>> originData = new HashMap<>();
+ List<TsFileResource> resources = new ArrayList<>();
+ for (int i = 1; i <= 5; i++) {
+ TsFileIOWriter writer =
+ new TsFileIOWriter(new File(dataDirectory,
String.format("%d-%d-0-0.tsfile", i, i)));
+ AlignedChunkWriterImpl alignedChunkWriter = new
AlignedChunkWriterImpl(schemas);
+ for (int j = i * 100; j < i * 100 + 100; j++) {
+ TsPrimitiveType[] values = {
+ new TsPrimitiveType.TsDouble(0.0D),
+ new TsPrimitiveType.TsFloat(0.0F),
+ null,
+ null,
+ new TsPrimitiveType.TsBinary(new Binary("")),
+ new TsPrimitiveType.TsBoolean(false)
+ };
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s0"), k -> new
ArrayList<>())
+ .add(new TimeValuePair(j, values[0]));
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s1"), k -> new
ArrayList<>())
+ .add(new TimeValuePair(j, values[1]));
+ originData.computeIfAbsent(new PartialPath("root.sg.d1.s2"), k ->
null);
+ originData.computeIfAbsent(new PartialPath("root.sg.d1.s3"), k ->
null);
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s4"), k -> new
ArrayList<>())
+ .add(new TimeValuePair(j, values[4]));
+ originData
+ .computeIfAbsent(new PartialPath("root.sg.d1.s5"), k -> new
ArrayList<>())
+ .add(new TimeValuePair(j, values[5]));
+ alignedChunkWriter.write(j, values);
+ }
+ writer.startChunkGroup("root.sg.d1");
+ alignedChunkWriter.writeToFileWriter(writer);
+ writer.endChunkGroup();
+ writer.endFile();
+ TsFileResource resource = new TsFileResource(writer.getFile(),
TsFileResourceStatus.NORMAL);
+ resource
+ .getModFile()
+ .write(new Deletion(new PartialPath("root.sg.d1.*"), i * 100, i *
100 + 20));
+ resource.getModFile().close();
+ int finalI = i;
+ originData.forEach(
+ (x, y) ->
+ y.removeIf(
+ timeValuePair ->
+ timeValuePair.getTimestamp() >= finalI * 100
+ && timeValuePair.getTimestamp() < finalI * 100 +
20));
+ resources.add(resource);
+ }
+ performer.setSourceFiles(resources);
+ TsFileResource targetResource =
+ TsFileNameGenerator.getInnerCompactionTargetFileResource(resources,
true);
+ performer.setTargetFiles(Collections.singletonList(targetResource));
+ performer.setSummary(new CompactionTaskSummary());
+ performer.perform();
+ Assert.assertTrue(targetResource.getTsFile().exists());
+ RestorableTsFileIOWriter checkWriter = new
RestorableTsFileIOWriter(targetResource.getTsFile());
+ Assert.assertFalse(checkWriter.hasCrashed());
+ }
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
index 6ba310d439..68320ee32f 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionCheckerUtils.java
@@ -545,6 +545,10 @@ public class CompactionCheckerUtils {
List<TimeValuePair> expectedTimeValueList = expectedData.get(path);
List<TimeValuePair> actualTimeValueList = actualData.get(path);
+ if (actualTimeValueList == null) {
+ assertNull(expectedTimeValueList);
+ continue;
+ }
assertEquals(expectedTimeValueList.size(), actualTimeValueList.size());
for (int i = 0; i < expectedTimeValueList.size(); ++i) {
