This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch delete_aligned in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 021e6547156582fb5fa77ec87fb2c72292fb0c45 Author: HTHou <[email protected]> AuthorDate: Tue Nov 23 21:06:09 2021 +0800 Fix deletion list logic --- .../iotdb/db/engine/memtable/AbstractMemTable.java | 8 ++--- .../engine/memtable/AlignedWritableMemChunk.java | 2 +- .../memtable/AlignedWritableMemChunkGroup.java | 8 ++--- .../apache/iotdb/db/engine/memtable/IMemTable.java | 6 ++-- .../db/engine/memtable/IWritableMemChunkGroup.java | 3 +- .../db/engine/memtable/WritableMemChunkGroup.java | 8 ++--- .../querycontext/AlignedReadOnlyMemChunk.java | 4 +-- .../db/engine/storagegroup/TsFileProcessor.java | 40 +--------------------- .../apache/iotdb/db/metadata/path/AlignedPath.java | 34 +++++++++++++++++- .../iotdb/db/metadata/path/MeasurementPath.java | 33 +++++++++++++++++- .../apache/iotdb/db/metadata/path/PartialPath.java | 22 +++++++++--- .../db/utils/datastructure/AlignedTVList.java | 37 +++++++++++++++++--- ...BRawQueryWithoutValueFilterWithDeletion2IT.java | 2 +- ...DBRawQueryWithoutValueFilterWithDeletionIT.java | 16 ++++----- 14 files changed, 145 insertions(+), 78 deletions(-) diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java index bbb6db0..3747841 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AbstractMemTable.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.engine.memtable; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.query.QueryProcessException; @@ -27,13 +28,12 @@ import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.TimeRange; +import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -335,9 +335,9 @@ public abstract class AbstractMemTable implements IMemTable { @Override public ReadOnlyMemChunk query( - PartialPath fullPath, long ttlLowerBound, List<TimeRange> deletionList) + PartialPath fullPath, long ttlLowerBound, List<Pair<Modification, IMemTable>> modsToMemtable) throws IOException, QueryProcessException { - return fullPath.getReadOnlyMemChunkFromMemTable(memTableMap, deletionList); + return fullPath.getReadOnlyMemChunkFromMemTable(this, modsToMemtable, ttlLowerBound); } @SuppressWarnings("squid:S3776") // high Cognitive Complexity diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java index caaa7fc..4ba016c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunk.java @@ -18,7 +18,6 @@ */ package org.apache.iotdb.db.engine.memtable; -import java.util.Set; import org.apache.iotdb.db.rescon.TVListAllocator; import org.apache.iotdb.db.utils.datastructure.AlignedTVList; import org.apache.iotdb.db.utils.datastructure.TVList; @@ -37,6 +36,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; public class AlignedWritableMemChunk implements IWritableMemChunk { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java index e56abf1b..2f98fab 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/AlignedWritableMemChunkGroup.java @@ -19,9 +19,6 @@ package org.apache.iotdb.db.engine.memtable; -import java.util.Iterator; -import java.util.Map.Entry; -import java.util.Set; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; @@ -29,6 +26,7 @@ import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { @@ -75,8 +73,8 @@ public class AlignedWritableMemChunkGroup implements IWritableMemChunkGroup { } @Override - public int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, - long endTimestamp) { + public int delete( + PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp) { int deletedPointsNumber = 0; boolean shouldDeleteAll = true; Set<String> measurements = memChunk.getAllMeasurements(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java index e05b1be..fa228f4 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IMemTable.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.engine.memtable; +import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.exception.metadata.MetadataException; @@ -25,7 +26,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan; import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan; -import org.apache.iotdb.tsfile.read.common.TimeRange; +import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import java.io.IOException; @@ -107,7 +108,8 @@ public interface IMemTable { void insertAlignedTablet(InsertTabletPlan insertTabletPlan, int start, int end) throws WriteProcessException; - ReadOnlyMemChunk query(PartialPath fullPath, long ttlLowerBound, List<TimeRange> deletionList) + ReadOnlyMemChunk query( + PartialPath fullPath, long ttlLowerBound, List<Pair<Modification, IMemTable>> modsToMemtable) throws IOException, QueryProcessException, MetadataException; /** putBack all the memory resources. */ diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java index 62b933e..00bcf7c 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/IWritableMemChunkGroup.java @@ -46,7 +46,8 @@ public interface IWritableMemChunkGroup { Map<String, IWritableMemChunk> getMemChunkMap(); - int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp); + int delete( + PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp); long getCurrentChunkPointNum(String measurement); } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java index 2f97021..50ac2f0 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/memtable/WritableMemChunkGroup.java @@ -19,15 +19,15 @@ package org.apache.iotdb.db.engine.memtable; -import java.util.Iterator; -import java.util.Map.Entry; import org.apache.iotdb.db.metadata.path.PartialPath; import org.apache.iotdb.tsfile.utils.BitMap; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Map.Entry; public class WritableMemChunkGroup implements IWritableMemChunkGroup { @@ -115,8 +115,8 @@ public class WritableMemChunkGroup implements IWritableMemChunkGroup { } @Override - public int delete(PartialPath originalPath, PartialPath devicePath, long startTimestamp, - long endTimestamp) { + public int delete( + PartialPath originalPath, PartialPath devicePath, long startTimestamp, long endTimestamp) { int deletedPointsNumber = 0; Iterator<Entry<String, IWritableMemChunk>> iter = memChunkMap.entrySet().iterator(); while (iter.hasNext()) { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/AlignedReadOnlyMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/AlignedReadOnlyMemChunk.java index 2f18ef7..1e058a1 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/AlignedReadOnlyMemChunk.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/AlignedReadOnlyMemChunk.java @@ -45,7 +45,7 @@ import java.util.List; public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { // deletion list for this chunk - private final List<TimeRange> deletionList; + private final List<List<TimeRange>> deletionList; private String measurementUid; private TSDataType dataType; @@ -68,7 +68,7 @@ public class AlignedReadOnlyMemChunk extends ReadOnlyMemChunk { * @param deletionList The timeRange of deletionList */ public AlignedReadOnlyMemChunk( - IMeasurementSchema schema, TVList tvList, int size, List<TimeRange> deletionList) + IMeasurementSchema schema, TVList tvList, int size, List<List<TimeRange>> deletionList) throws IOException, QueryProcessException { super(); this.measurementUid = schema.getMeasurementId(); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java index 5908597..e52a43d 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java @@ -60,7 +60,6 @@ import org.apache.iotdb.service.rpc.thrift.TSStatus; import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; -import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.utils.Binary; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; @@ -1227,41 +1226,6 @@ public class TsFileProcessor { return storageGroupName; } - /** get modifications from a memtable */ - private List<Modification> getModificationsForMemtable(IMemTable memTable) { - List<Modification> modifications = new ArrayList<>(); - boolean foundMemtable = false; - for (Pair<Modification, IMemTable> entry : modsToMemtable) { - if (foundMemtable || entry.right.equals(memTable)) { - modifications.add(entry.left); - foundMemtable = true; - } - } - return modifications; - } - - /** - * construct a deletion list from a memtable - * - * @param memTable memtable - * @param timeLowerBound time water mark - */ - private List<TimeRange> constructDeletionList( - IMemTable memTable, PartialPath fullPath, long timeLowerBound) { - List<TimeRange> deletionList = new ArrayList<>(); - deletionList.add(new TimeRange(Long.MIN_VALUE, timeLowerBound)); - for (Modification modification : getModificationsForMemtable(memTable)) { - if (modification instanceof Deletion) { - Deletion deletion = (Deletion) modification; - if (deletion.getPath().matchFullPath(fullPath) && deletion.getEndTime() > timeLowerBound) { - long lowerBound = Math.max(deletion.getStartTime(), timeLowerBound); - deletionList.add(new TimeRange(lowerBound, deletion.getEndTime())); - } - } - } - return TimeRange.sortAndMerge(deletionList); - } - /** * get the chunk(s) in the memtable (one from work memtable and the other ones in flushing * memtables and then compact them into one TimeValuePairSorter). Then get the related @@ -1284,10 +1248,8 @@ public class TsFileProcessor { if (flushingMemTable.isSignalMemTable()) { continue; } - List<TimeRange> deletionList = - constructDeletionList(flushingMemTable, fullPath, context.getQueryTimeLowerBound()); ReadOnlyMemChunk memChunk = - flushingMemTable.query(fullPath, context.getQueryTimeLowerBound(), deletionList); + flushingMemTable.query(fullPath, context.getQueryTimeLowerBound(), modsToMemtable); if (memChunk != null) { readOnlyMemChunks.add(memChunk); } diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java index 44457ad..323178a 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/AlignedPath.java @@ -21,7 +21,9 @@ package org.apache.iotdb.db.metadata.path; import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunk; import org.apache.iotdb.db.engine.memtable.AlignedWritableMemChunkGroup; +import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup; +import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.querycontext.AlignedReadOnlyMemChunk; @@ -47,6 +49,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.schema.VectorMeasurementSchema; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; @@ -353,8 +356,12 @@ public class AlignedPath extends PartialPath { @Override public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( - Map<String, IWritableMemChunkGroup> memTableMap, List<TimeRange> deletionList) + IMemTable memTable, List<Pair<Modification, IMemTable>> modsToMemtable, long timeLowerBound) throws QueryProcessException, IOException { + + List<List<TimeRange>> deletionList = + constructDeletionList(memTable, modsToMemtable, timeLowerBound); + Map<String, IWritableMemChunkGroup> memTableMap = memTable.getMemTableMap(); // check If memtable contains this path if (!memTableMap.containsKey(getDevice())) { return null; @@ -378,6 +385,31 @@ public class AlignedPath extends PartialPath { getMeasurementSchema(), alignedTvListCopy, curSize, deletionList); } + private List<List<TimeRange>> constructDeletionList( + IMemTable memTable, List<Pair<Modification, IMemTable>> modsToMemtable, long timeLowerBound) { + if (modsToMemtable == null) { + return null; + } + List<List<TimeRange>> deletionList = new ArrayList<>(); + for (String measurement : measurementList) { + List<TimeRange> columnDeletionList = new ArrayList<>(); + columnDeletionList.add(new TimeRange(Long.MIN_VALUE, timeLowerBound)); + for (Modification modification : getModificationsForMemtable(memTable, modsToMemtable)) { + if (modification instanceof Deletion) { + Deletion deletion = (Deletion) modification; + PartialPath fullPath = this.concatNode(measurement); + if (deletion.getPath().matchFullPath(fullPath) + && deletion.getEndTime() > timeLowerBound) { + long lowerBound = Math.max(deletion.getStartTime(), timeLowerBound); + columnDeletionList.add(new TimeRange(lowerBound, deletion.getEndTime())); + } + } + } + deletionList.add(TimeRange.sortAndMerge(columnDeletionList)); + } + return deletionList; + } + @Override public List<IChunkMetadata> getVisibleMetadataListFromWriter( RestorableTsFileIOWriter writer, TsFileResource tsFileResource, QueryContext context) { diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java index 9d4277a..2d96dfb 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/MeasurementPath.java @@ -19,8 +19,10 @@ package org.apache.iotdb.db.metadata.path; import org.apache.iotdb.db.conf.IoTDBConstant; +import org.apache.iotdb.db.engine.memtable.IMemTable; import org.apache.iotdb.db.engine.memtable.IWritableMemChunk; import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup; +import org.apache.iotdb.db.engine.modification.Deletion; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; @@ -41,6 +43,7 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics; import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; @@ -247,8 +250,11 @@ public class MeasurementPath extends PartialPath { @Override public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( - Map<String, IWritableMemChunkGroup> memTableMap, List<TimeRange> deletionList) + IMemTable memTable, List<Pair<Modification, IMemTable>> modsToMemtable, long timeLowerBound) throws QueryProcessException, IOException { + + List<TimeRange> deletionList = constructDeletionList(memTable, modsToMemtable, timeLowerBound); + Map<String, IWritableMemChunkGroup> memTableMap = memTable.getMemTableMap(); // check If Memtable Contains this path if (!memTableMap.containsKey(getDevice()) || !memTableMap.get(getDevice()).contains(getMeasurement())) { @@ -269,6 +275,31 @@ public class MeasurementPath extends PartialPath { deletionList); } + /** + * construct a deletion list from a memtable + * + * @param memTable memtable + * @param timeLowerBound time water mark + */ + private List<TimeRange> constructDeletionList( + IMemTable memTable, List<Pair<Modification, IMemTable>> modsToMemtable, long timeLowerBound) { + if (modsToMemtable == null) { + return null; + } + List<TimeRange> deletionList = new ArrayList<>(); + deletionList.add(new TimeRange(Long.MIN_VALUE, timeLowerBound)); + for (Modification modification : getModificationsForMemtable(memTable, modsToMemtable)) { + if (modification instanceof Deletion) { + Deletion deletion = (Deletion) modification; + if (deletion.getPath().matchFullPath(this) && deletion.getEndTime() > timeLowerBound) { + long lowerBound = Math.max(deletion.getStartTime(), timeLowerBound); + deletionList.add(new TimeRange(lowerBound, deletion.getEndTime())); + } + } + } + return TimeRange.sortAndMerge(deletionList); + } + @Override public MeasurementPath clone() { MeasurementPath newMeasurementPath = null; diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java index ae42486..e0a7cce 100644 --- a/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java +++ b/server/src/main/java/org/apache/iotdb/db/metadata/path/PartialPath.java @@ -18,7 +18,8 @@ */ package org.apache.iotdb.db.metadata.path; -import org.apache.iotdb.db.engine.memtable.IWritableMemChunkGroup; +import org.apache.iotdb.db.engine.memtable.IMemTable; +import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.querycontext.QueryDataSource; import org.apache.iotdb.db.engine.querycontext.ReadOnlyMemChunk; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; @@ -35,8 +36,8 @@ import org.apache.iotdb.tsfile.common.constant.TsFileConstant; import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata; import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; import org.apache.iotdb.tsfile.read.common.Path; -import org.apache.iotdb.tsfile.read.common.TimeRange; import org.apache.iotdb.tsfile.read.filter.basic.Filter; +import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema; import org.apache.iotdb.tsfile.write.writer.RestorableTsFileIOWriter; @@ -48,7 +49,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -423,11 +423,25 @@ public class PartialPath extends Path implements Comparable<Path>, Cloneable { * @return ReadOnlyMemChunk */ public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable( - Map<String, IWritableMemChunkGroup> memTableMap, List<TimeRange> deletionList) + IMemTable memTable, List<Pair<Modification, IMemTable>> modsToMemtable, long timeLowerBound) throws QueryProcessException, IOException { throw new UnsupportedOperationException("Should call exact sub class!"); } + /** get modifications from a memtable */ + protected List<Modification> getModificationsForMemtable( + IMemTable memTable, List<Pair<Modification, IMemTable>> modsToMemtable) { + List<Modification> modifications = new ArrayList<>(); + boolean foundMemtable = false; + for (Pair<Modification, IMemTable> entry : modsToMemtable) { + if (foundMemtable || entry.right.equals(memTable)) { + modifications.add(entry.left); + foundMemtable = true; + } + } + return modifications; + } + @Override public PartialPath clone() { return new PartialPath(this.getNodes().clone()); diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java index 9ce81f0..cfff238 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AlignedTVList.java @@ -888,22 +888,35 @@ public class AlignedTVList extends TVList { } public IPointReader getAlignedIterator( - int floatPrecision, List<TSEncoding> encodingList, int size, List<TimeRange> deletionList) { + int floatPrecision, + List<TSEncoding> encodingList, + int size, + List<List<TimeRange>> deletionList) { return new AlignedIte(floatPrecision, encodingList, size, deletionList); } private class AlignedIte extends Ite { private List<TSEncoding> encodingList; + private int[] deleteCursors; + /** this field is effective only in the AlignedTvlist in a AlignedRealOnlyMemChunk. */ + private List<List<TimeRange>> deletionList; public AlignedIte() { super(); } public AlignedIte( - int floatPrecision, List<TSEncoding> encodingList, int size, List<TimeRange> deletionList) { - super(floatPrecision, null, size, deletionList); + int floatPrecision, + List<TSEncoding> encodingList, + int size, + List<List<TimeRange>> deletionList) { + super(floatPrecision, null, size, null); this.encodingList = encodingList; + this.deletionList = deletionList; + if (deletionList != null) { + deleteCursors = new int[deletionList.size()]; + } } @Override @@ -915,7 +928,7 @@ public class AlignedTVList extends TVList { List<Integer> timeDuplicatedAlignedRowIndexList = null; while (cur < iteSize) { long time = getTime(cur); - if (isPointDeleted(time) || (cur + 1 < size() && (time == getTime(cur + 1)))) { + if (cur + 1 < size() && (time == getTime(cur + 1))) { if (timeDuplicatedAlignedRowIndexList == null) { timeDuplicatedAlignedRowIndexList = new ArrayList<>(); timeDuplicatedAlignedRowIndexList.add(getValueIndex(cur)); @@ -933,6 +946,7 @@ public class AlignedTVList extends TVList { } else { tvPair = getTimeValuePair(cur, time, floatPrecision, encodingList); } + deletePointsInDeletionList(time, tvPair); cur++; if (tvPair.getValue() != null) { cachedTimeValuePair = tvPair; @@ -943,5 +957,20 @@ public class AlignedTVList extends TVList { return false; } + + private void deletePointsInDeletionList(long timestamp, TimeValuePair tvPair) { + if (deletionList == null) { + return; + } + for (int i = 0; i < deleteCursors.length; i++) { + while (deletionList.get(i) != null && deleteCursors[i] < deletionList.get(i).size()) { + if (deletionList.get(i).get(deleteCursors[i]).contains(timestamp)) { + tvPair.getValue().getVector()[i] = null; + } else if (deletionList.get(i).get(deleteCursors[i]).getMax() < timestamp) { + deleteCursors[i]++; + } + } + } + } } } diff --git a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java index f3377d3..64a269f 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletion2IT.java @@ -60,7 +60,7 @@ public class IoTDBRawQueryWithoutValueFilterWithDeletion2IT Statement statement = connection.createStatement()) { // TODO currently aligned data in memory doesn't support deletion, so we flush all data to // disk before doing deletion - statement.execute("flush"); + // statement.execute("flush"); statement.execute("delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); } catch (Exception e) { diff --git a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java index efa1b4c..b694cd2 100644 --- a/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java +++ b/server/src/test/java/org/apache/iotdb/db/integration/aligned/IoTDBRawQueryWithoutValueFilterWithDeletionIT.java @@ -66,11 +66,9 @@ public class IoTDBRawQueryWithoutValueFilterWithDeletionIT { DriverManager.getConnection( Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root"); Statement statement = connection.createStatement()) { - // TODO currently aligned data in memory doesn't support deletion, so we flush all data to - // disk before doing deletion - statement.execute("flush"); statement.execute("delete timeseries root.sg1.d1.s2"); statement.execute("delete from root.sg1.d1.s1 where time <= 21"); + statement.execute("delete from root.sg1.d1.s5 where time <= 31"); } catch (Exception e) { e.printStackTrace(); } @@ -273,11 +271,11 @@ public class IoTDBRawQueryWithoutValueFilterWithDeletionIT { String[] retArray = new String[] { - "9,null,9,false,aligned_test9", - "10,null,10,true,aligned_test10", + "9,null,9,false,null", + "10,null,10,true,null", "11,null,11,null,null", "12,null,12,null,null", - "13,null,130000,true,aligned_unseq_test13", + "13,null,130000,true,null", "14,null,14,null,null", "15,null,15,null,null", "16,null,16,null,null", @@ -295,7 +293,7 @@ public class IoTDBRawQueryWithoutValueFilterWithDeletionIT { "28,null,28,false,null", "29,null,29,false,null", "30,null,30,false,null", - "31,null,null,null,aligned_test31", + "31,null,null,null,null", "32,null,null,null,aligned_test32", "33,null,null,null,aligned_test33", }; @@ -494,7 +492,7 @@ public class IoTDBRawQueryWithoutValueFilterWithDeletionIT { "28,null,false,null", "29,null,false,null", "30,null,false,null", - "31,null,null,aligned_test31", + "31,null,null,null", "32,null,null,aligned_test32", "33,null,null,aligned_test33", "34,null,null,aligned_test34", @@ -559,7 +557,7 @@ public class IoTDBRawQueryWithoutValueFilterWithDeletionIT { "28,null,false,null,null,false,null", "29,null,false,null,null,false,null", "30,null,false,null,null,false,null", - "31,non_aligned_test31,null,null,aligned_test31,null,null", + "31,non_aligned_test31,null,null,null,null,null", "32,non_aligned_test32,null,null,aligned_test32,null,null", "33,non_aligned_test33,null,null,aligned_test33,null,null", "34,non_aligned_test34,null,null,aligned_test34,null,null",
