This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.1 by this push:
new ef8b427e797 [To rel/1.1] Fix TsfileResource error after delete device
in sequence working memtable (#10052)
ef8b427e797 is described below
commit ef8b427e797c071603a4794f8329f23501435c4c
Author: Haonan <[email protected]>
AuthorDate: Tue Jun 6 01:05:08 2023 +0800
[To rel/1.1] Fix TsfileResource error after delete device in sequence
working memtable (#10052)
---
.../db/engine/storagegroup/TsFileProcessor.java | 15 +++++++++++-
.../db/engine/storagegroup/TsFileResource.java | 13 +++++++++--
.../db/engine/settle/SettleRequestHandlerTest.java | 1 -
.../db/engine/storagegroup/DataRegionTest.java | 27 ++++++++++++++++++++++
4 files changed, 52 insertions(+), 4 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 3d399ecdff6..b6806a99c86 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -680,6 +680,11 @@ public class TsFileProcessor {
}
try {
if (workMemTable != null) {
+ logger.info(
+ "[Deletion] Deletion with path: {}, time:{}-{} in workMemTable",
+ deletion.getPath(),
+ deletion.getStartTime(),
+ deletion.getEndTime());
for (PartialPath device : devicePaths) {
workMemTable.delete(
deletion.getPath(), device, deletion.getStartTime(),
deletion.getEndTime());
@@ -953,7 +958,15 @@ public class TsFileProcessor {
Map<String, Long> lastTimeForEachDevice = new HashMap<>();
if (sequence) {
lastTimeForEachDevice = tobeFlushed.getMaxTime();
- tsFileResource.updateEndTime(lastTimeForEachDevice);
+ // If some devices have been removed in MemTable, the number of device
in MemTable and
+ // tsFileResource will not be the same. And the endTime of these devices
in resource will be
+ // Long.minValue.
+ // In the case, we need to delete the removed devices in tsFileResource.
+ if (lastTimeForEachDevice.size() != tsFileResource.getDevices().size()) {
+
tsFileResource.deleteRemovedDeviceAndUpdateEndTime(lastTimeForEachDevice);
+ } else {
+ tsFileResource.updateEndTime(lastTimeForEachDevice);
+ }
}
for (FlushListener flushListener : flushListeners) {
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index e08ba2c9982..c3cea8210e5 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -1112,8 +1112,17 @@ public class TsFileResource {
}
}
- public void updateEndTime(Map<String, Long> times) {
- for (Map.Entry<String, Long> entry : times.entrySet()) {
+ public void deleteRemovedDeviceAndUpdateEndTime(Map<String, Long>
lastTimeForEachDevice) {
+ ITimeIndex newTimeIndex = CONFIG.getTimeIndexLevel().getTimeIndex();
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
+ newTimeIndex.updateStartTime(entry.getKey(),
timeIndex.getStartTime(entry.getKey()));
+ newTimeIndex.updateEndTime(entry.getKey(), entry.getValue());
+ }
+ timeIndex = newTimeIndex;
+ }
+
+ public void updateEndTime(Map<String, Long> lastTimeForEachDevice) {
+ for (Map.Entry<String, Long> entry : lastTimeForEachDevice.entrySet()) {
timeIndex.updateEndTime(entry.getKey(), entry.getValue());
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
index f849ac9f63d..e832df7b398 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/settle/SettleRequestHandlerTest.java
@@ -143,7 +143,6 @@ public class SettleRequestHandlerTest {
}
for (TsFileProcessor tsFileProcessor :
dataRegion.getWorkSequenceTsFileProcessors()) {
paths.add(tsFileProcessor.getTsFileResource().getTsFilePath());
- tsFileProcessor.syncFlush();
}
dataRegion.syncCloseAllWorkingTsFileProcessors();
if (i != 2) {
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
index 34b2655e6cc..066404ddb06 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/DataRegionTest.java
@@ -1118,6 +1118,33 @@ public class DataRegionTest {
Assert.assertEquals(3,
tsFileResource.getModFile().getModifications().size());
}
+ @Test
+ public void testDeleteDataInSeqWorkingMemtable()
+ throws IllegalPathException, WriteProcessException, IOException {
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d0");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId,
String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ for (int j = 100; j < 200; j++) {
+ TSRecord record = new TSRecord(j, "root.vehicle.d199");
+ record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId,
String.valueOf(j)));
+ dataRegion.insert(buildInsertRowNodeByTSRecord(record));
+ }
+ TsFileResource tsFileResource =
dataRegion.getTsFileManager().getTsFileList(true).get(0);
+
+ // delete data which is not in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d0.s0"), 50, 99,
0, null);
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d200.s0"), 50, 70,
0, null);
+
+ // delete data which is in working memtable
+ dataRegion.deleteByDevice(new PartialPath("root.vehicle.d199.*"), 50, 500,
0, null);
+
+ dataRegion.syncCloseAllWorkingTsFileProcessors();
+ Assert.assertFalse(tsFileResource.getModFile().exists());
+
Assert.assertFalse(tsFileResource.getDevices().contains("root.vehicle.d199"));
+ }
+
@Test
public void testFlushingEmptyMemtable()
throws IllegalPathException, WriteProcessException, IOException {
