This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch fix_delete_device in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4c4dd6e41545e98945ec128922ff1e131a587443 Author: HTHou <[email protected]> AuthorDate: Mon Jun 5 18:46:25 2023 +0800 fix tefileResource error after delete device in sequence working memtable --- .../db/engine/storagegroup/TsFileProcessor.java | 15 +++++++++++- .../db/engine/storagegroup/TsFileResource.java | 13 +++++++++-- .../db/engine/storagegroup/DataRegionTest.java | 27 ++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 3d399ecdff6..b6806a99c86 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -680,6 +680,11 @@ public class TsFileProcessor { } try { if (workMemTable != null) { + logger.info( + "[Deletion] Deletion with path: {}, time:{}-{} in workMemTable", + deletion.getPath(), + deletion.getStartTime(), + deletion.getEndTime()); for (PartialPath device : devicePaths) { workMemTable.delete( deletion.getPath(), device, deletion.getStartTime(), deletion.getEndTime()); @@ -953,7 +958,15 @@ public class TsFileProcessor { Map<String, Long> lastTimeForEachDevice = new HashMap<>(); if (sequence) { lastTimeForEachDevice = tobeFlushed.getMaxTime(); - tsFileResource.updateEndTime(lastTimeForEachDevice); + // If some devices have been removed in MemTable, the number of device in MemTable and + // tsFileResource will not be the same. And the endTime of these devices in resource will be + // Long.minValue. + // In the case, we need to delete the removed devices in tsFileResource. + if (lastTimeForEachDevice.size() != tsFileResource.getDevices().size()) { + tsFileResource.deleteRemovedDeviceAndUpdateEndTime(lastTimeForEachDevice); + } else { + tsFileResource.updateEndTime(lastTimeForEachDevice); + } } for (FlushListener flushListener : flushListeners) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java index e08ba2c9982..c3cea8210e5 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java @@ -1112,8 +1112,17 @@ public class TsFileResource { } } - public void updateEndTime(Map<String, Long> times) { - for (Map.Entry<String, Long> entry : times.entrySet()) { + public void deleteRemovedDeviceAndUpdateEndTime(Map<String, Long> lastTimeForEachDevice) { + ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex(); + for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) { + newTimeIndex.updateStartTime(entry.getKey(), timeIndex.getStartTime(entry.getKey())); + newTimeIndex.updateEndTime(entry.getKey(), entry.getValue()); + } + timeIndex = newTimeIndex; + } + + public void updateEndTime(Map<String, Long> lastTimeForEachDevice) { + for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) { timeIndex.updateEndTime(entry.getKey(), entry.getValue()); } } diff --git a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java index 34b2655e6cc..066404ddb06 100644 --- a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java +++ b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java @@ -1118,6 +1118,33 @@ public class DataRegionTest { Assert.assertEquals(3, tsFileResource.getModFile().getModifications().size()); } + @Test + public void testDeleteDataInSeqWorkingMemtable() + throws IllegalPathException, WriteProcessException, IOException { + for (int j = 100; j < 200; j++) { + TSRecord record = new TSRecord(j, "root.vehicle.d0"); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + dataRegion.insert(buildInsertRowNodeByTSRecord(record)); + } + for (int j = 100; j < 200; j++) { + TSRecord record = new TSRecord(j, "root.vehicle.d199"); + record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(j))); + dataRegion.insert(buildInsertRowNodeByTSRecord(record)); + } + TsFileResource tsFileResource = dataRegion.getTsFileManager().getTsFileList(true).get(0); + + // delete data which is not in working memtable + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99, 0, null); + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70, 0, null); + + // delete data which is in working memtable + dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500, 0, null); + + dataRegion.syncCloseAllWorkingTsFileProcessors(); + Assert.assertFalse(tsFileResource.getModFile().exists()); + Assert.assertFalse(tsFileResource.getDevices().contains("root.vehicle.d199")); + } + @Test public void testFlushingEmptyMemtable() throws IllegalPathException, WriteProcessException, IOException {
