This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 4a9f055ed1 [To rel/0.12]Avoid serializing resource file and adding
mods to file that don't contain the device when deleting data (#6629)
4a9f055ed1 is described below
commit 4a9f055ed18b7bbe954d1ad00d4c8cec6546d87e
Author: 周沛辰 <[email protected]>
AuthorDate: Wed Jul 13 09:06:16 2022 +0800
[To rel/0.12]Avoid serializing resource file and adding mods to file that
don't contain the device when deleting data (#6629)
---
.../engine/storagegroup/StorageGroupProcessor.java | 15 ++++---
.../db/engine/storagegroup/TsFileResource.java | 24 +++++-----
.../storagegroup/StorageGroupProcessorTest.java | 52 ++++++++++++++++++++++
3 files changed, 75 insertions(+), 16 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 376125d510..cfc4630484 100755
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1815,15 +1815,20 @@ public class StorageGroupProcessor {
Set<PartialPath> devicePaths,
long deleteStart,
long deleteEnd) {
+ if (!tsFileResource.isClosed()) {
+ // tsfile is not closed
+ return false;
+ }
for (PartialPath device : devicePaths) {
String deviceId = device.getFullPath();
- long endTime = tsFileResource.getEndTime(deviceId);
- if (endTime == Long.MIN_VALUE) {
- return false;
+ if (!tsFileResource.getDevices().contains(deviceId)) {
+ // resource does not contain this device
+ continue;
}
- if (tsFileResource.getDevices().contains(deviceId)
- && (deleteEnd >= tsFileResource.getStartTime(deviceId) &&
deleteStart <= endTime)) {
+ if (deleteEnd >= tsFileResource.getStartTime(deviceId)
+ && deleteStart <= tsFileResource.getEndTime(deviceId)) {
+ // time range of device has overlap with the deletion
return false;
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 3a5af2c4db..7e5d14552e 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -823,17 +823,19 @@ public class TsFileResource {
if (planIndex == Long.MIN_VALUE || planIndex == Long.MAX_VALUE) {
return;
}
- maxPlanIndex = Math.max(maxPlanIndex, planIndex);
- minPlanIndex = Math.min(minPlanIndex, planIndex);
- if (closed) {
- try {
- serialize();
- } catch (IOException e) {
- LOGGER.error(
- "Cannot serialize TsFileResource {} when updating plan index
{}-{}",
- this,
- maxPlanIndex,
- planIndex);
+ if (planIndex < minPlanIndex || planIndex > maxPlanIndex) {
+ maxPlanIndex = Math.max(maxPlanIndex, planIndex);
+ minPlanIndex = Math.min(minPlanIndex, planIndex);
+ if (closed) {
+ try {
+ serialize();
+ } catch (IOException e) {
+ LOGGER.error(
+ "Cannot serialize TsFileResource {} when updating plan index
{}-{}",
+ this,
+ maxPlanIndex,
+ planIndex);
+ }
}
}
}
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
index 7214a7b2d6..1321da94ba 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessorTest.java
@@ -819,6 +819,58 @@ public class StorageGroupProcessorTest {
config.setCloseTsFileIntervalAfterFlushing(prevCloseTsFileInterval);
}
+ /**
+ * Totally 5 tsfiles<br>
+ * file 0, file 2 and file 4 has d0 ~ d1, time range is 0 ~ 99, 200 ~ 299,
400 ~ 499<br>
+ * file 1, file 3 has d0 ~ d2, time range is 100 ~ 199, 300 ~ 399<br>
+ * delete d2 in time range 50 ~ 150 and 150 ~ 450. Therefore, only file 1
and file 3 has mods.
+ */
+ @Test
+ public void testDeleteDataNotInFile()
+ throws IllegalPathException, WriteProcessException,
InterruptedException, IOException {
+ for (int i = 0; i < 5; i++) {
+ if (i % 2 == 0) {
+ for (int d = 0; d < 2; d++) {
+ for (int count = i * 100; count < i * 100 + 100; count++) {
+ TSRecord record = new TSRecord(count, "root.vehicle.d" + d);
+ record.addTuple(
+ DataPoint.getDataPoint(TSDataType.INT32, measurementId,
String.valueOf(count)));
+ insertToStorageGroupProcessor(record);
+ }
+ }
+ } else {
+ for (int d = 0; d < 3; d++) {
+ for (int count = i * 100; count < i * 100 + 100; count++) {
+ TSRecord record = new TSRecord(count, "root.vehicle.d" + d);
+ record.addTuple(
+ DataPoint.getDataPoint(TSDataType.INT32, measurementId,
String.valueOf(count)));
+ insertToStorageGroupProcessor(record);
+ }
+ }
+ }
+ processor.syncCloseAllWorkingTsFileProcessors();
+ }
+
+ // delete root.vehicle.d2.s0 data in the second file
+ processor.delete(new PartialPath("root.vehicle.d2.s0"), 50, 150, 0);
+
+ // delete root.vehicle.d2.s0 data in the third file
+ processor.delete(new PartialPath("root.vehicle.d2.s0"), 150, 450, 0);
+
+ for (int i = 0; i < processor.getSequenceFileTreeSet().size(); i++) {
+ TsFileResource resource = processor.getSequenceFileTreeSet().get(i);
+ if (i == 1) {
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(2,
resource.getModFile().getModifications().size());
+ } else if (i == 3) {
+ Assert.assertTrue(resource.getModFile().exists());
+ Assert.assertEquals(1,
resource.getModFile().getModifications().size());
+ } else {
+ Assert.assertFalse(resource.getModFile().exists());
+ }
+ }
+ }
+
class DummySGP extends StorageGroupProcessor {
DummySGP(String systemInfoDir, String storageGroupName) throws
StorageGroupProcessorException {
