This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_delete_resource_error_1.2 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9893884fa38e6540d2463a8de0e9e118cf063698 Author: Haonan <[email protected]> AuthorDate: Tue Jun 6 01:05:08 2023 +0800 Fix TsfileResource error after delete device in sequence working memtable --- .../db/engine/storagegroup/TsFileProcessor.java | 15 +++++++++++- .../db/engine/storagegroup/TsFileResource.java | 13 +++++++++-- .../db/engine/settle/SettleRequestHandlerTest.java | 1 - .../db/engine/storagegroup/DataRegionTest.java | 27 ++++++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 656e30b40f1..f4bdddea394 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -710,6 +710,11 @@ public class TsFileProcessor { } try { if (workMemTable != null) { + logger.info( + "[Deletion] Deletion with path: {}, time:{}-{} in workMemTable", + deletion.getPath(), + deletion.getStartTime(), + deletion.getEndTime()); for (PartialPath device : devicePaths) { workMemTable.delete( deletion.getPath(), device, deletion.getStartTime(), deletion.getEndTime()); @@ -987,7 +992,15 @@ public class TsFileProcessor { Map<String, Long> lastTimeForEachDevice = new HashMap<>(); if (sequence) { lastTimeForEachDevice = tobeFlushed.getMaxTime(); - tsFileResource.updateEndTime(lastTimeForEachDevice); + // If some devices have been removed in MemTable, the number of device in MemTable and + // tsFileResource will not be the same. And the endTime of these devices in resource will be + // Long.minValue. + // In the case, we need to delete the removed devices in tsFileResource. + if (lastTimeForEachDevice.size() != tsFileResource.getDevices().size()) { + tsFileResource.deleteRemovedDeviceAndUpdateEndTime(lastTimeForEachDevice); + } else { + tsFileResource.updateEndTime(lastTimeForEachDevice); + } } for (FlushListener flushListener : flushListeners) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index e6c430da223..bdb93bbd165 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -1192,8 +1192,17 @@ public class TsFileResource { } } - public void updateEndTime(Map<String, Long> times) { - for (Map.Entry<String, Long> entry : times.entrySet()) { + public void deleteRemovedDeviceAndUpdateEndTime(Map<String, Long> lastTimeForEachDevice) { + ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex(); + for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) { + newTimeIndex.updateStartTime(entry.getKey(), timeIndex.getStartTime(entry.getKey())); + newTimeIndex.updateEndTime(entry.getKey(), entry.getValue()); + } + timeIndex = newTimeIndex; + } + + public void updateEndTime(Map<String, Long> lastTimeForEachDevice) { + for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) { timeIndex.updateEndTime(entry.getKey(), entry.getValue()); } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java index f849ac9f63d..e832df7b398 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java @@ -143,7 +143,6 @@ public class SettleRequestHandlerTest { } for (TsFileProcessor tsFileProcessor : dataRegion.getWorkSequenceTsFileProcessors()) { paths.add(tsFileProcessor.getTsFileResource().getTsFilePath()); - tsFileProcessor.syncFlush(); } dataRegion.syncCloseAllWorkingTsFileProcessors(); if (i != 2) { diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java index 89406f8f580..acf7e6eb672 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java @@ -1118,6 +1118,33 @@ public class DataRegionTest { Assert.assertEquals(3, tsFileResource.getModFile().getModifications().size()); } + @Test + public void testDeleteDataInSeqWorkingMemtable() + throws IllegalPathException, WriteProcessException, IOException { + for (int j = 100; j < 200; j++) { + TSRecord record = new TSRecord(j, "root.vehicle.d0"); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + dataRegion.insert(buildInsertRowNodeByTSRecord(record)); + } + for (int j = 100; j < 200; j++) { + TSRecord record = new TSRecord(j, "root.vehicle.d199"); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + dataRegion.insert(buildInsertRowNodeByTSRecord(record)); + } + TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + + // delete data which is not in working memtable + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); + + // delete data which is in working memtable + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0, null); + + dataRegion.syncCloseAllWorkingTsFileProcessors(); + Assert.assertFalse(tsFileResource.getModFile().exists()); + Assert.assertFalse(tsFileResource.getDevices().contains("root.vehicle.d199")); + } + @Test public void testFlushingEmptyMemtable() throws IllegalPathException, WriteProcessException, IOException {
