This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch delete_aligned in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2706b61b30cd89faba6e9f11365073e8f97d3497 Author: HTHou <[email protected]> AuthorDate: Tue Nov 23 16:33:59 2021 +0800 init --- .../iotdb/db/engine/memtable/AbstractMemTable.java | 19 +------------- .../engine/memtable/AlignedWritableMemChunk.java | 8 ++++-- .../memtable/AlignedWritableMemChunkGroup.java | 29 ++++++++++++++++++++++ .../db/engine/memtable/IWritableMemChunk.java | 2 +- .../db/engine/memtable/IWritableMemChunkGroup.java | 3 +++ .../db/engine/memtable/WritableMemChunkGroup.java | 25 +++++++++++++++++++ .../db/utils/datastructure/AlignedTVList.java | 13 ++++++++-- ...IoTDBRawQueryWithValueFilterWithDeletionIT.java | 2 +- 8 files changed, 77 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index c5e8ae2..bbb6db0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@ -348,24 +348,7 @@ public abstract class AbstractMemTable implements IMemTable { if (memChunkGroup == null) { return; } - - Iterator<Entry<String, IWritableMemChunk>> iter = - memChunkGroup.getMemChunkMap().entrySet().iterator(); - while (iter.hasNext()) { - Entry<String, IWritableMemChunk> entry = iter.next(); - IWritableMemChunk chunk = entry.getValue(); - // the key is measurement rather than component of multiMeasurement - PartialPath fullPath = devicePath.concatNode(entry.getKey()); - if (originalPath.matchFullPath(fullPath)) { - // matchFullPath ensures this branch could work on delete data of unary or multi measurement - // and delete timeseries or aligned timeseries - if (startTimestamp == Long.MIN_VALUE && endTimestamp == Long.MAX_VALUE) { - iter.remove(); - } - int deletedPointsNumber = chunk.delete(startTimestamp, endTimestamp); - totalPointsNum -= deletedPointsNumber; - } - } + totalPointsNum -= memChunkGroup.delete(originalPath, devicePath, startTimestamp, endTimestamp); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java index 001a3c2..caaa7fc 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.engine.memtable; +import java.util.Set; import org.apache.iotdb.db.rescon.TVListAllocator; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; @@ -56,6 +57,10 @@ public class AlignedWritableMemChunk implements IWritableMemChunk { this.list = TVListAllocator.getInstance().allocate(dataTypeList); } + public Set<String> getAllMeasurements() { + return measurementIndexMap.keySet(); + } + public boolean containsMeasurement(String measurementId) { return measurementIndexMap.containsKey(measurementId); } @@ -242,9 +247,8 @@ public class AlignedWritableMemChunk implements IWritableMemChunk { } @Override - // TODO: THIS METHOLD IS FOR DELETING ONE COLUMN OF A VECTOR public int delete(long lowerBound, long upperBound, String measurementId) { - return 0; + return list.delete(lowerBound, upperBound, measurementIndexMap.get(measurementId)); } @Override diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java index 0a62bd6..e56abf1b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java @@ -19,6 +19,10 @@ package org.apache.iotdb.db.engine.memtable; +import java.util.Iterator; +import java.util.Map.Entry; +import java.util.Set; +import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; @@ -71,6 +75,31 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { } @Override + public int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, + long endTimestamp) { + int deletedPointsNumber = 0; + boolean shouldDeleteAll = true; + Set<String> measurements = memChunk.getAllMeasurements(); + for (String measurement : measurements) { + PartialPath fullPath = devicePath.concatNode(measurement); + if (!originalPath.matchFullPath(fullPath)) { + shouldDeleteAll = false; + } + } + if (shouldDeleteAll) { + deletedPointsNumber += memChunk.delete(startTimestamp, endTimestamp); + return deletedPointsNumber; + } + for (String measurement : measurements) { + PartialPath fullPath = devicePath.concatNode(measurement); + if (originalPath.matchFullPath(fullPath)) { + deletedPointsNumber += memChunk.delete(startTimestamp, endTimestamp, measurement); + } + } + return deletedPointsNumber; + } + + @Override public long getCurrentChunkPointNum(String measurement) { return memChunk.count(); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java index c2c1781..e17980b 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunk.java @@ -124,7 +124,7 @@ public interface IWritableMemChunk { /** @return how many points are deleted */ int delete(long lowerBound, long upperBound); - // For delete one column in the vector + /** @return how many points are deleted */ int delete(long lowerBound, long upperBound, String measurementId); IChunkWriter createIChunkWriter(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java index 85586aa..62b933e 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.memtable; +import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; @@ -45,5 +46,7 @@ public interface IWritableMemChunkGroup { Map<String, IWritableMemChunk> getMemChunkMap(); + int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp); + long getCurrentChunkPointNum(String measurement); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java index 2ca32ce..2f97021 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java @@ -19,6 +19,9 @@ package org.apache.iotdb.db.engine.memtable; +import java.util.Iterator; +import java.util.Map.Entry; +import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; @@ -112,6 +115,28 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup { } @Override + public int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, + long endTimestamp) { + int deletedPointsNumber = 0; + Iterator<Entry<String, IWritableMemChunk>> iter = memChunkMap.entrySet().iterator(); + while (iter.hasNext()) { + Entry<String, IWritableMemChunk> entry = iter.next(); + IWritableMemChunk chunk = entry.getValue(); + // the key is measurement rather than component of multiMeasurement + PartialPath fullPath = devicePath.concatNode(entry.getKey()); + if (originalPath.matchFullPath(fullPath)) { + // matchFullPath ensures this branch could work on delete data of unary or multi measurement + // and delete timeseries + if (startTimestamp == Long.MIN_VALUE && endTimestamp == Long.MAX_VALUE) { + iter.remove(); + } + deletedPointsNumber += chunk.delete(startTimestamp, endTimestamp); + } + } + return deletedPointsNumber; + } + + @Override public long getCurrentChunkPointNum(String measurement) { return memChunkMap.get(measurement).count(); } diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 1c9f6ed..9ce81f0 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -453,9 +453,18 @@ public class AlignedTVList extends TVList { return deletedNumber * getTsDataTypes().size(); } - // TODO: THIS METHOLD IS FOR DELETING ONE COLUMN OF A VECTOR public int delete(long lowerBound, long upperBound, int columnIndex) { - throw new UnsupportedOperationException(ERR_DATATYPE_NOT_CONSISTENT); + int deletedNumber = 0; + for (int i = 0; i < size; i++) { + long time = getTime(i); + if (time >= lowerBound && time <= upperBound) { + int arrayIndex = i / ARRAY_SIZE; + int elementIndex = i % ARRAY_SIZE; + markNullValue(columnIndex, arrayIndex, elementIndex); + deletedNumber++; + } + } + return deletedNumber; } protected void set(int index, long timestamp, int value) { diff --git a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java index a1ce64c..b61bcf1 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithValueFilterWithDeletionIT.java @@ -68,7 +68,7 @@ public class IoTDBRawQueryWithValueFilterWithDeletionIT { Statement statement = connection.createStatement()) { // TODO currently aligned data in memory doesn't support deletion, so we flush all data to // disk before doing deletion - statement.execute("flush"); + // statement.execute("flush"); statement.execute("delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); } catch (Exception e) {
