This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 212cebf4e21 Fix TsFileResource is deleted cause compaction validation
result is not correct (#11878)
212cebf4e21 is described below
commit 212cebf4e21060ca955bf00c6ccda33777d3f354
Author: shuwenwei <[email protected]>
AuthorDate: Mon Jan 15 15:24:17 2024 +0800
Fix TsFileResource is deleted cause compaction validation result is not
correct (#11878)
---
.../execute/task/InnerSpaceCompactionTask.java | 2 +
.../compaction/schedule/CompactionWorker.java | 3 +
.../dataregion/tsfile/TsFileResource.java | 26 ++++--
.../dataregion/utils/TsFileResourceUtils.java | 75 +++++++--------
.../compaction/CompactionValidationTest.java | 103 +++++++++++++++++++++
5 files changed, 162 insertions(+), 47 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 503894f9869..0f46242093c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -489,6 +489,8 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
memoryCost =
innerSpaceEstimator.estimateInnerCompactionMemory(selectedTsFileResourceList);
} catch (IOException e) {
innerSpaceEstimator.cleanup();
+ LOGGER.error("Meet error when estimate inner compaction memory", e);
+ return -1;
}
}
return memoryCost;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionWorker.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionWorker.java
index 9daa908e6ce..677ba75b647 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionWorker.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionWorker.java
@@ -87,6 +87,9 @@ public class CompactionWorker implements Runnable {
task.transitSourceFilesToMerging();
if
(IoTDBDescriptor.getInstance().getConfig().isEnableCompactionMemControl()) {
estimatedMemoryCost = task.getEstimatedMemoryCost();
+ if (estimatedMemoryCost < 0) {
+ return false;
+ }
CompactionTaskType taskType = task.getCompactionTaskType();
memoryAcquired =
SystemInfo.getInstance().addCompactionMemoryCost(taskType,
estimatedMemoryCost, 60);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 06b23a2e650..2baabaf0c6e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -400,17 +400,23 @@ public class TsFileResource {
public DeviceTimeIndex buildDeviceTimeIndex() throws IOException {
readLock();
- try (InputStream inputStream =
- FSFactoryProducer.getFSFactory().getBufferedInputStream(file.getPath()
+ RESOURCE_SUFFIX)) {
- ReadWriteIOUtils.readByte(inputStream);
- ITimeIndex timeIndexFromResourceFile =
ITimeIndex.createTimeIndex(inputStream);
- if (!(timeIndexFromResourceFile instanceof DeviceTimeIndex)) {
- throw new IOException("cannot build DeviceTimeIndex from resource " +
file.getPath());
+ try {
+ if (!resourceFileExists()) {
+ throw new IOException("resource file not found");
+ }
+ try (InputStream inputStream =
+ FSFactoryProducer.getFSFactory()
+ .getBufferedInputStream(file.getPath() + RESOURCE_SUFFIX)) {
+ ReadWriteIOUtils.readByte(inputStream);
+ ITimeIndex timeIndexFromResourceFile =
ITimeIndex.createTimeIndex(inputStream);
+ if (!(timeIndexFromResourceFile instanceof DeviceTimeIndex)) {
+ throw new IOException("cannot build DeviceTimeIndex from resource "
+ file.getPath());
+ }
+ return (DeviceTimeIndex) timeIndexFromResourceFile;
+ } catch (Exception e) {
+ throw new IOException(
+ "Can't read file " + file.getPath() + RESOURCE_SUFFIX + " from
disk", e);
}
- return (DeviceTimeIndex) timeIndexFromResourceFile;
- } catch (Exception e) {
- throw new IOException(
- "Can't read file " + file.getPath() + RESOURCE_SUFFIX + " from
disk", e);
} finally {
readUnlock();
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java
index eeae1731f68..7f98e96ba5d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileResourceUtils.java
@@ -339,49 +339,50 @@ public class TsFileResourceUtils {
}
public static boolean
validateTsFileResourcesHasNoOverlap(List<TsFileResource> resources) {
- try {
- // deviceID -> <TsFileResource, last end time>
- Map<String, Pair<TsFileResource, Long>> lastEndTimeMap = new HashMap<>();
- for (TsFileResource resource : resources) {
- DeviceTimeIndex timeIndex;
- if (resource.getTimeIndexType() != 1) {
- // if time index is not device time index, then deserialize it from
resource file
+ // deviceID -> <TsFileResource, last end time>
+ Map<String, Pair<TsFileResource, Long>> lastEndTimeMap = new HashMap<>();
+ for (TsFileResource resource : resources) {
+ DeviceTimeIndex timeIndex;
+ if (resource.getTimeIndexType() != 1) {
+ // if time index is not device time index, then deserialize it from
resource file
+ try {
timeIndex = resource.buildDeviceTimeIndex();
- } else {
- timeIndex = (DeviceTimeIndex) resource.getTimeIndex();
+ } catch (IOException e) {
+ // skip such files
+ continue;
}
- if (timeIndex == null) {
+ } else {
+ timeIndex = (DeviceTimeIndex) resource.getTimeIndex();
+ }
+ if (timeIndex == null) {
+ return false;
+ }
+ Set<String> devices = timeIndex.getDevices();
+ for (String device : devices) {
+ long currentStartTime = timeIndex.getStartTime(device);
+ long currentEndTime = timeIndex.getEndTime(device);
+ Pair<TsFileResource, Long> lastDeviceInfo =
+ lastEndTimeMap.computeIfAbsent(device, x -> new Pair<>(null,
Long.MIN_VALUE));
+ long lastEndTime = lastDeviceInfo.right;
+ if (lastEndTime >= currentStartTime) {
+ logger.error(
+ "Device {} is overlapped between {} and {}, "
+ + "end time in {} is {}, start time in {} is {}",
+ device,
+ lastDeviceInfo.left,
+ resource,
+ lastDeviceInfo.left,
+ lastEndTime,
+ resource,
+ currentStartTime);
return false;
}
- Set<String> devices = timeIndex.getDevices();
- for (String device : devices) {
- long currentStartTime = timeIndex.getStartTime(device);
- long currentEndTime = timeIndex.getEndTime(device);
- Pair<TsFileResource, Long> lastDeviceInfo =
- lastEndTimeMap.computeIfAbsent(device, x -> new Pair<>(null,
Long.MIN_VALUE));
- long lastEndTime = lastDeviceInfo.right;
- if (lastEndTime >= currentStartTime) {
- logger.error(
- "Device {} is overlapped between {} and {}, "
- + "end time in {} is {}, start time in {} is {}",
- device,
- lastDeviceInfo.left,
- resource,
- lastDeviceInfo.left,
- lastEndTime,
- resource,
- currentStartTime);
- return false;
- }
- lastDeviceInfo.left = resource;
- lastDeviceInfo.right = currentEndTime;
- lastEndTimeMap.put(device, lastDeviceInfo);
- }
+ lastDeviceInfo.left = resource;
+ lastDeviceInfo.right = currentEndTime;
+ lastEndTimeMap.put(device, lastDeviceInfo);
}
- return true;
- } catch (IOException e) {
- return true;
}
+ return true;
}
public static void updateTsFileResource(
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java
index 7438e8eda4f..29c5670119a 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionValidationTest.java
@@ -20,6 +20,8 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
+import
org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex;
import org.apache.iotdb.db.storageengine.dataregion.utils.TsFileResourceUtils;
import org.apache.iotdb.db.utils.constant.TestConstant;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
@@ -43,6 +45,7 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
public class CompactionValidationTest {
@@ -206,4 +209,104 @@ public class CompactionValidationTest {
TsFileResourceUtils.validateTsFileDataCorrectness(mockTsFile);
}
}
+
+ @Test
+ public void
testTsFileResourceIsDeletedByOtherCompactionTaskWhenValidateOverlap1() {
+ TsFileResource resource1 = new TsFileResource();
+ resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile"));
+ resource1.setTimeIndex(new DeviceTimeIndex());
+ resource1.updateStartTime("d1", 1);
+ resource1.updateEndTime("d1", 2);
+
+ TsFileResource resource2 = new TsFileResource();
+ resource2.setFile(new File(dir + File.separator + "2-2-0-0.tsfile"));
+ resource2.setTimeIndex(new FileTimeIndex());
+
+ TsFileResource resource3 = new TsFileResource();
+ resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile"));
+ resource3.setTimeIndex(new DeviceTimeIndex());
+ resource3.updateStartTime("d1", 4);
+ resource3.updateEndTime("d1", 5);
+
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ Arrays.asList(resource1, resource2, resource3)));
+ }
+
+ @Test
+ public void
testTsFileResourceIsDeletedByOtherCompactionTaskWhenValidateOverlap2() {
+ TsFileResource resource1 = new TsFileResource();
+ resource1.setTimeIndex(new DeviceTimeIndex());
+ resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile"));
+ resource1.updateStartTime("d1", 1);
+ resource1.updateEndTime("d1", 2);
+
+ TsFileResource resource2 = new TsFileResource();
+ resource2.setFile(new File(dir + File.separator + "2-2-0-0.tsfile"));
+ resource2.setTimeIndex(new FileTimeIndex());
+
+ TsFileResource resource3 = new TsFileResource();
+ resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile"));
+ resource3.setTimeIndex(new DeviceTimeIndex());
+ resource3.updateStartTime("d1", 1);
+ resource3.updateEndTime("d1", 5);
+
+ Assert.assertFalse(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ Arrays.asList(resource1, resource2, resource3)));
+ }
+
+ @Test
+ public void testTsFileResourceIsBrokenWhenValidateOverlap1() throws
IOException {
+ TsFileResource resource1 = new TsFileResource();
+ resource1.setTimeIndex(new DeviceTimeIndex());
+ resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile"));
+ resource1.updateStartTime("d1", 1);
+ resource1.updateEndTime("d1", 2);
+
+ TsFileResource resource2 = new TsFileResource();
+ File tsFile2 = new File(dir + File.separator + "2-2-0-0.tsfile");
+ Assert.assertTrue(tsFile2.createNewFile());
+ File resourceFile2 = new File(dir + File.separator +
"2-2-0-0.tsfile.resource");
+ Assert.assertTrue(resourceFile2.createNewFile());
+ resource2.setFile(tsFile2);
+ resource2.setTimeIndex(new FileTimeIndex());
+
+ TsFileResource resource3 = new TsFileResource();
+ resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile"));
+ resource3.setTimeIndex(new DeviceTimeIndex());
+ resource3.updateStartTime("d1", 4);
+ resource3.updateEndTime("d1", 5);
+
+ Assert.assertTrue(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ Arrays.asList(resource1, resource2, resource3)));
+ }
+
+ @Test
+ public void testTsFileResourceIsBrokenWhenValidateOverlap2() throws
IOException {
+ TsFileResource resource1 = new TsFileResource();
+ resource1.setTimeIndex(new DeviceTimeIndex());
+ resource1.setFile(new File(dir + File.separator + "1-1-0-0.tsfile"));
+ resource1.updateStartTime("d1", 1);
+ resource1.updateEndTime("d1", 2);
+
+ TsFileResource resource2 = new TsFileResource();
+ File tsFile2 = new File(dir + File.separator + "2-2-0-0.tsfile");
+ Assert.assertTrue(tsFile2.createNewFile());
+ File resourceFile2 = new File(dir + File.separator +
"2-2-0-0.tsfile.resource");
+ Assert.assertTrue(resourceFile2.createNewFile());
+ resource2.setFile(tsFile2);
+ resource2.setTimeIndex(new FileTimeIndex());
+
+ TsFileResource resource3 = new TsFileResource();
+ resource3.setFile(new File(dir + File.separator + "3-3-0-0.tsfile"));
+ resource3.setTimeIndex(new DeviceTimeIndex());
+ resource3.updateStartTime("d1", 1);
+ resource3.updateEndTime("d1", 5);
+
+ Assert.assertFalse(
+ TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(
+ Arrays.asList(resource1, resource2, resource3)));
+ }
}
