This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch force_ci/support_schema_evolution in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 5f64c454499aadceb6e3c9050c36017cf6c704d8 Author: Tian Jiang <[email protected]> AuthorDate: Thu Dec 25 17:45:46 2025 +0800 support delete with sevo --- .../impl/DataNodeInternalRPCServiceImpl.java | 4 +- .../db/queryengine/plan/analyze/AnalyzeUtils.java | 38 ++-- .../db/storageengine/dataregion/DataRegion.java | 172 +++++++-------- .../compaction/execute/utils/CompactionUtils.java | 2 +- .../dataregion/modification/DeletionPredicate.java | 40 ++-- .../{IDPredicate.java => TagPredicate.java} | 46 ++-- .../dataregion/tsfile/evolution/EvolvedSchema.java | 38 +++- .../dataregion/tsfile/fileset/TsFileSet.java | 2 +- .../file/UnsealedTsFileRecoverPerformer.java | 2 +- .../iotdb/db/metadata/path/PatternTreeMapTest.java | 2 +- .../db/pipe/consensus/DeletionRecoverTest.java | 2 +- .../db/pipe/consensus/DeletionResourceTest.java | 2 +- .../PipePlanTablePatternParseVisitorTest.java | 8 +- .../node/write/RelationalDeleteDataNodeTest.java | 8 +- .../storageengine/dataregion/DataRegionTest.java | 243 ++++++++++++++++----- .../tablemodel/CompactionWithAllNullRowsTest.java | 12 +- .../modification/ModificationFileTest.java | 6 +- .../modification/TableDeletionEntryTest.java | 8 +- 18 files changed, 398 insertions(+), 237 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index b1fb833a38e..84e3e17a498 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -195,7 +195,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.Compacti import org.apache.iotdb.db.storageengine.dataregion.compaction.settle.SettleRequestHandler; import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.SchemaEvolution; import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; @@ -1989,7 +1989,7 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface new TableDeletionEntry( new DeletionPredicate( req.getTableName(), - new IDPredicate.NOP(), + new TagPredicate.NOP(), Collections.singletonList(req.getColumnName())), new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE)), // the request is only sent to associated region diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java index 2e0b0b52fdc..331c18d54c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeUtils.java @@ -53,9 +53,9 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.schemaengine.table.DataNodeTreeViewSchemaUtils; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.And; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -436,23 +436,23 @@ public class AnalyzeUtils { Queue<Expression> expressionQueue = new LinkedList<>(); expressionQueue.add(expression); DeletionPredicate predicate = new DeletionPredicate(table.getTableName()); - IDPredicate idPredicate = null; + TagPredicate tagPredicate = null; TimeRange timeRange = new TimeRange(Long.MIN_VALUE, Long.MAX_VALUE, true); while (!expressionQueue.isEmpty()) { Expression currExp = expressionQueue.remove(); if (currExp instanceof LogicalExpression) { parseAndPredicate(((LogicalExpression) currExp), expressionQueue); } else if (currExp instanceof ComparisonExpression) { - idPredicate = - parseComparison(((ComparisonExpression) currExp), timeRange, idPredicate, table); + tagPredicate = + parseComparison(((ComparisonExpression) currExp), timeRange, tagPredicate, table); } else if (currExp instanceof IsNullPredicate) { - idPredicate = parseIsNull((IsNullPredicate) currExp, idPredicate, table); + tagPredicate = parseIsNull((IsNullPredicate) currExp, tagPredicate, table); } else { throw new SemanticException("Unsupported expression: " + currExp + " in " + expression); } } - if (idPredicate != null) { - predicate.setIdPredicate(idPredicate); + if (tagPredicate != null) { + predicate.setIdPredicate(tagPredicate); } if (timeRange.getStartTime() > timeRange.getEndTime()) { throw new SemanticException( @@ -472,8 +472,8 @@ public class AnalyzeUtils { expressionQueue.addAll(expression.getTerms()); } - private static IDPredicate parseIsNull( - IsNullPredicate isNullPredicate, IDPredicate oldPredicate, TsTable table) { + private static TagPredicate parseIsNull( + IsNullPredicate isNullPredicate, TagPredicate oldPredicate, TsTable table) { Expression leftHandExp = isNullPredicate.getValue(); if (!(leftHandExp instanceof Identifier)) { throw new SemanticException("Left hand expression is not an identifier: " + leftHandExp); @@ -486,25 +486,25 @@ public class AnalyzeUtils { } // the first segment is the table name, so + 1 - IDPredicate newPredicate = new SegmentExactMatch(null, idColumnOrdinal + 1); + TagPredicate newPredicate = new SegmentExactMatch(null, idColumnOrdinal + 1); return combinePredicates(oldPredicate, newPredicate); } - private static IDPredicate combinePredicates(IDPredicate oldPredicate, IDPredicate newPredicate) { + private static TagPredicate combinePredicates(TagPredicate oldPredicate, TagPredicate newPredicate) { if (oldPredicate == null) { return newPredicate; } - if (oldPredicate instanceof IDPredicate.And) { + if (oldPredicate instanceof TagPredicate.And) { ((And) oldPredicate).add(newPredicate); return oldPredicate; } - return new IDPredicate.And(oldPredicate, newPredicate); + return new TagPredicate.And(oldPredicate, newPredicate); } - private static IDPredicate parseComparison( + private static TagPredicate parseComparison( ComparisonExpression comparisonExpression, TimeRange timeRange, - IDPredicate oldPredicate, + TagPredicate oldPredicate, TsTable table) { Expression left = comparisonExpression.getLeft(); Expression right = comparisonExpression.getRight(); @@ -556,11 +556,11 @@ public class AnalyzeUtils { "The column '" + columnName + "' does not exist or is not a tag column"); } - IDPredicate newPredicate = getIdPredicate(comparisonExpression, right, idColumnOrdinal); + TagPredicate newPredicate = getIdPredicate(comparisonExpression, right, idColumnOrdinal); return combinePredicates(oldPredicate, newPredicate); } - private static IDPredicate getIdPredicate( + private static TagPredicate getIdPredicate( ComparisonExpression comparisonExpression, Expression right, int idColumnOrdinal) { if (comparisonExpression.getOperator() != ComparisonExpression.Operator.EQUAL) { throw new SemanticException("The operator of tag predicate must be '=' for " + right); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index c37b84d22b4..835a0977fa5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -772,7 +772,7 @@ public class DataRegion implements IDataRegionForQuery { @SuppressWarnings("OptionalGetWithoutIsPresent") // checked above long endTime = resource.getEndTime(deviceId).get(); if (mergedEvolvedSchema != null) { - deviceId = mergedEvolvedSchema.rewriteDeviceId(deviceId); + deviceId = mergedEvolvedSchema.rewriteToOriginal(deviceId); } endTimeMap.put(deviceId, endTime); } @@ -794,7 +794,7 @@ public class DataRegion implements IDataRegionForQuery { //noinspection OptionalGetWithoutIsPresent long endTime = resource.getEndTime(deviceId).get(); if (mergedEvolvedSchema != null) { - deviceId = mergedEvolvedSchema.rewriteDeviceId(deviceId); + deviceId = mergedEvolvedSchema.rewriteToOriginal(deviceId); } endTimeMap.put(deviceId, endTime); } @@ -1270,12 +1270,12 @@ public class DataRegion implements IDataRegionForQuery { private void renameTableForObjects(String nameBefore, String nameAfter) { // TODO-SchemaEvolution - throw new UnsupportedOperationException(); + // throw new UnsupportedOperationException(); } private void renameMeasurementForObjects(String tableName, String nameBefore, String nameAfter) { // TODO-SchemaEvolution - throw new UnsupportedOperationException(); + // throw new UnsupportedOperationException(); } /** @@ -2681,7 +2681,7 @@ public class DataRegion implements IDataRegionForQuery { EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(); IDeviceID deviceIdBackThen = singleDeviceId; if (evolvedSchema != null) { - deviceIdBackThen = evolvedSchema.rewriteDeviceId(singleDeviceId); + deviceIdBackThen = evolvedSchema.rewriteToOriginal(singleDeviceId); } if (!tsFileResource.isSatisfied( @@ -3020,6 +3020,11 @@ public class DataRegion implements IDataRegionForQuery { return false; } + EvolvedSchema evolvedSchema = tsFileResource.getMergedEvolvedSchema(); + if (evolvedSchema != null) { + deletion = evolvedSchema.rewriteToOriginal(deletion); + } + for (IDeviceID device : tsFileResource.getDevices()) { // we are iterating the time index so the times are definitely present long startTime = tsFileResource.getTimeIndex().getStartTime(device).get(); @@ -3071,10 +3076,70 @@ public class DataRegion implements IDataRegionForQuery { } } + private boolean canBeFullyDeleted(ArrayDeviceTimeIndex deviceTimeIndex, TableDeletionEntry tableDeletionEntry) { + Set<IDeviceID> devicesInFile = deviceTimeIndex.getDevices(); + String tableName = tableDeletionEntry.getTableName(); + long matchSize = + devicesInFile.stream() + .filter( + device -> { + if (logger.isDebugEnabled()) { + logger.debug( + "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", + device, + device.getTableName(), + tableDeletionEntry.getPredicate().matches(device)); + } + return tableName.equals(device.getTableName()) + && tableDeletionEntry.getPredicate().matches(device); + }) + .count(); + boolean onlyOneTable = matchSize == devicesInFile.size(); + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, matchSize is {}, onlyOneTable is {}", + tableName, + matchSize, + onlyOneTable); + } + + if (onlyOneTable) { + matchSize = 0; + for (IDeviceID device : devicesInFile) { + Optional<Long> optStart = deviceTimeIndex.getStartTime(device); + Optional<Long> optEnd = deviceTimeIndex.getEndTime(device); + if (!optStart.isPresent() || !optEnd.isPresent()) { + continue; + } + + long fileStartTime = optStart.get(); + long fileEndTime = optEnd.get(); + + if (logger.isDebugEnabled()) { + logger.debug( + "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", + device.getTableName(), + device, + tableDeletionEntry.getStartTime(), + tableDeletionEntry.getEndTime(), + fileStartTime, + fileEndTime); + } + if (isFileFullyMatchedByTime(tableDeletionEntry, fileStartTime, fileEndTime)) { + ++matchSize; + } else { + return false; + } + } + return matchSize == devicesInFile.size(); + } else { + return false; + } + } + private void deleteDataInSealedFiles(Collection<TsFileResource> sealedTsFiles, ModEntry deletion) throws IOException { - Set<ModificationFile> involvedModificationFiles = new HashSet<>(); - List<TsFileResource> deletedByMods = new ArrayList<>(); + Set<Pair<ModificationFile, ModEntry>> involvedModificationFiles = new HashSet<>(); List<TsFileResource> deletedByFiles = new ArrayList<>(); for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { @@ -3082,96 +3147,23 @@ public class DataRegion implements IDataRegionForQuery { } ITimeIndex timeIndex = sealedTsFile.getTimeIndex(); + EvolvedSchema evolvedSchema = sealedTsFile.getMergedEvolvedSchema(); if ((timeIndex instanceof ArrayDeviceTimeIndex) && (deletion.getType() == ModType.TABLE_DELETION)) { ArrayDeviceTimeIndex deviceTimeIndex = (ArrayDeviceTimeIndex) timeIndex; - Set<IDeviceID> devicesInFile = deviceTimeIndex.getDevices(); - boolean onlyOneTable = false; - - if (deletion instanceof TableDeletionEntry) { - TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; - String tableName = tableDeletionEntry.getTableName(); - long matchSize = - devicesInFile.stream() - .filter( - device -> { - if (logger.isDebugEnabled()) { - logger.debug( - "device is {}, deviceTable is {}, tableDeletionEntry.getPredicate().matches(device) is {}", - device, - device.getTableName(), - tableDeletionEntry.getPredicate().matches(device)); - } - return tableName.equals(device.getTableName()) - && tableDeletionEntry.getPredicate().matches(device); - }) - .count(); - onlyOneTable = matchSize == devicesInFile.size(); - if (logger.isDebugEnabled()) { - logger.debug( - "tableName is {}, matchSize is {}, onlyOneTable is {}", - tableName, - matchSize, - onlyOneTable); - } - } - - if (onlyOneTable) { - int matchSize = 0; - for (IDeviceID device : devicesInFile) { - Optional<Long> optStart = deviceTimeIndex.getStartTime(device); - Optional<Long> optEnd = deviceTimeIndex.getEndTime(device); - if (!optStart.isPresent() || !optEnd.isPresent()) { - continue; - } - - long fileStartTime = optStart.get(); - long fileEndTime = optEnd.get(); - - if (logger.isDebugEnabled()) { - logger.debug( - "tableName is {}, device is {}, deletionStartTime is {}, deletionEndTime is {}, fileStartTime is {}, fileEndTime is {}", - device.getTableName(), - device, - deletion.getStartTime(), - deletion.getEndTime(), - fileStartTime, - fileEndTime); - } - if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)) { - ++matchSize; - } else { - deletedByMods.add(sealedTsFile); - break; - } - } - if (matchSize == devicesInFile.size()) { - deletedByFiles.add(sealedTsFile); - } - - if (logger.isDebugEnabled()) { - logger.debug("expect is {}, actual is {}", devicesInFile.size(), matchSize); - for (TsFileResource tsFileResource : deletedByFiles) { - logger.debug( - "delete tsFileResource is {}", tsFileResource.getTsFile().getAbsolutePath()); - } - } + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + tableDeletionEntry = evolvedSchema != null? evolvedSchema.rewriteToOriginal(tableDeletionEntry) : tableDeletionEntry; + if (canBeFullyDeleted(deviceTimeIndex, tableDeletionEntry)) { + deletedByFiles.add(sealedTsFile); } else { - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + involvedModificationFiles.add(new Pair<>(sealedTsFile.getModFileForWrite(), tableDeletionEntry)); } } else { - involvedModificationFiles.add(sealedTsFile.getModFileForWrite()); + involvedModificationFiles.add(new Pair<>(sealedTsFile.getModFileForWrite(), evolvedSchema != null? evolvedSchema.rewriteToOriginal(deletion) : deletion)); } } - for (TsFileResource tsFileResource : deletedByMods) { - if (tsFileResource.isClosed() - || !tsFileResource.getProcessor().deleteDataInMemory(deletion)) { - involvedModificationFiles.add(tsFileResource.getModFileForWrite()); - } // else do nothing - } - if (!deletedByFiles.isEmpty()) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { @@ -3188,10 +3180,10 @@ public class DataRegion implements IDataRegionForQuery { List<Exception> exceptions = involvedModificationFiles.parallelStream() .map( - modFile -> { + modFileEntryPair -> { try { - modFile.write(deletion); - modFile.close(); + modFileEntryPair.getLeft().write(modFileEntryPair.getRight()); + modFileEntryPair.getLeft().close(); } catch (Exception e) { return e; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java index 4f58e8fc132..e6ec34e529b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java @@ -33,7 +33,7 @@ import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModFileManagement; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index 7e79e8f580d..da9617a7e67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -18,7 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; @@ -42,7 +42,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(DeletionPredicate.class); private String tableName; - private IDPredicate idPredicate = new NOP(); + private TagPredicate tagPredicate = new NOP(); // an empty list means affecting all columns private List<String> measurementNames = Collections.emptyList(); @@ -52,24 +52,24 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable this.tableName = tableName; } - public DeletionPredicate(String tableName, IDPredicate idPredicate) { + public DeletionPredicate(String tableName, TagPredicate tagPredicate) { this.tableName = tableName; - this.idPredicate = idPredicate; + this.tagPredicate = tagPredicate; } public DeletionPredicate( - String tableName, IDPredicate idPredicate, List<String> measurementNames) { + String tableName, TagPredicate tagPredicate, List<String> measurementNames) { this.tableName = tableName; - this.idPredicate = idPredicate; + this.tagPredicate = tagPredicate; this.measurementNames = measurementNames; } public boolean matches(IDeviceID deviceID) { - return tableName.equals(deviceID.getTableName()) && idPredicate.matches(deviceID); + return tableName.equals(deviceID.getTableName()) && tagPredicate.matches(deviceID); } - public void setIdPredicate(IDPredicate idPredicate) { - this.idPredicate = idPredicate; + public void setIdPredicate(TagPredicate tagPredicate) { + this.tagPredicate = tagPredicate; } public String getTableName() { @@ -80,6 +80,10 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable return measurementNames; } + public TagPredicate getTagPredicate() { + return tagPredicate; + } + public boolean affects(String measurementName) { return measurementNames.isEmpty() || measurementNames.contains(measurementName); } @@ -87,7 +91,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable @Override public long serialize(OutputStream stream) throws IOException { long size = ReadWriteIOUtils.writeVar(tableName, stream); - size += idPredicate.serialize(stream); + size += tagPredicate.serialize(stream); size += ReadWriteForEncodingUtils.writeVarInt(measurementNames.size(), stream); for (String measurementName : measurementNames) { size += ReadWriteIOUtils.writeVar(measurementName, stream); @@ -98,7 +102,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable @Override public long serialize(ByteBuffer buffer) { long size = ReadWriteIOUtils.writeVar(tableName, buffer); - size += idPredicate.serialize(buffer); + size += tagPredicate.serialize(buffer); size += ReadWriteForEncodingUtils.writeVarInt(measurementNames.size(), buffer); for (String measurementName : measurementNames) { size += ReadWriteIOUtils.writeVar(measurementName, buffer); @@ -109,7 +113,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable @Override public void deserialize(InputStream stream) throws IOException { tableName = ReadWriteIOUtils.readVarIntString(stream); - idPredicate = IDPredicate.createFrom(stream); + tagPredicate = TagPredicate.createFrom(stream); int measurementLength = ReadWriteForEncodingUtils.readVarInt(stream); if (measurementLength > 0) { @@ -125,7 +129,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable @Override public void deserialize(ByteBuffer buffer) { tableName = ReadWriteIOUtils.readVarIntString(buffer); - idPredicate = IDPredicate.createFrom(buffer); + tagPredicate = TagPredicate.createFrom(buffer); int measurementLength = ReadWriteForEncodingUtils.readVarInt(buffer); if (measurementLength > 0) { @@ -143,7 +147,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable int size = ReadWriteForEncodingUtils.varIntSize(tableName.length()) + tableName.length() * Character.BYTES - + idPredicate.serializedSize() + + tagPredicate.serializedSize() + ReadWriteForEncodingUtils.varIntSize(measurementNames.size()); for (String measurementName : measurementNames) { size += @@ -163,13 +167,13 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable } DeletionPredicate that = (DeletionPredicate) o; return Objects.equals(tableName, that.tableName) - && Objects.equals(idPredicate, that.idPredicate) + && Objects.equals(tagPredicate, that.tagPredicate) && Objects.equals(measurementNames, that.measurementNames); } @Override public int hashCode() { - return Objects.hash(tableName, idPredicate, measurementNames); + return Objects.hash(tableName, tagPredicate, measurementNames); } @Override @@ -179,7 +183,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable + tableName + '\'' + ", idPredicate=" - + idPredicate + + tagPredicate + ", measurementNames=" + measurementNames + '}'; @@ -189,7 +193,7 @@ public class DeletionPredicate implements StreamSerializable, BufferSerializable public long ramBytesUsed() { return SHALLOW_SIZE + RamUsageEstimator.sizeOf(tableName) - + RamUsageEstimator.sizeOfObject(idPredicate) + + RamUsageEstimator.sizeOfObject(tagPredicate) + RamUsageEstimator.sizeOfArrayList(measurementNames); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java similarity index 89% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java index 44741f9e679..8aca6ed5fe7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/IDPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/TagPredicate.java @@ -18,6 +18,7 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.EvolvedSchema; import org.apache.iotdb.db.utils.io.BufferSerializable; import org.apache.iotdb.db.utils.io.StreamSerializable; @@ -38,7 +39,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -public abstract class IDPredicate implements StreamSerializable, BufferSerializable, Accountable { +public abstract class TagPredicate implements StreamSerializable, BufferSerializable, Accountable { public int serializedSize() { // type @@ -73,12 +74,16 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa protected final IDPredicateType type; - protected IDPredicate(IDPredicateType type) { + protected TagPredicate(IDPredicateType type) { this.type = type; } public abstract boolean matches(IDeviceID deviceID); + public TagPredicate rewriteToOriginal(EvolvedSchema evolvedSchema) { + return this; + } + @Override public long serialize(OutputStream stream) throws IOException { return type.serialize(stream); @@ -89,9 +94,9 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa return type.serialize(buffer); } - public static IDPredicate createFrom(ByteBuffer buffer) { + public static TagPredicate createFrom(ByteBuffer buffer) { IDPredicateType type = IDPredicateType.deserialize(buffer); - IDPredicate predicate; + TagPredicate predicate; if (Objects.requireNonNull(type) == IDPredicateType.NOP) { predicate = new NOP(); } else if (Objects.requireNonNull(type) == IDPredicateType.FULL_EXACT_MATCH) { @@ -107,9 +112,9 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa return predicate; } - public static IDPredicate createFrom(InputStream stream) throws IOException { + public static TagPredicate createFrom(InputStream stream) throws IOException { IDPredicateType type = IDPredicateType.deserialize(stream); - IDPredicate predicate; + TagPredicate predicate; if (Objects.requireNonNull(type) == IDPredicateType.NOP) { predicate = new NOP(); } else if (Objects.requireNonNull(type) == IDPredicateType.FULL_EXACT_MATCH) { @@ -125,7 +130,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa return predicate; } - public static class NOP extends IDPredicate { + public static class NOP extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(NOP.class); public NOP() { @@ -168,7 +173,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa } } - public static class FullExactMatch extends IDPredicate { + public static class FullExactMatch extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(FullExactMatch.class); @@ -243,9 +248,14 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public long ramBytesUsed() { return SHALLOW_SIZE + RamUsageEstimator.sizeOfObject(deviceID); } + + @Override + public TagPredicate rewriteToOriginal(EvolvedSchema evolvedSchema) { + return new FullExactMatch(evolvedSchema.rewriteToOriginal(deviceID)); + } } - public static class SegmentExactMatch extends IDPredicate { + public static class SegmentExactMatch extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(SegmentExactMatch.class); @@ -342,17 +352,17 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa } } - public static class And extends IDPredicate { + public static class And extends TagPredicate { public static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(And.class); - private final List<IDPredicate> predicates = new ArrayList<>(); + private final List<TagPredicate> predicates = new ArrayList<>(); - public And(IDPredicate... predicates) { + public And(TagPredicate... predicates) { super(IDPredicateType.AND); Collections.addAll(this.predicates, predicates); } - public void add(IDPredicate predicate) { + public void add(TagPredicate predicate) { predicates.add(predicate); } @@ -360,7 +370,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public int serializedSize() { int serializedSize = super.serializedSize(); serializedSize += ReadWriteForEncodingUtils.varIntSize(predicates.size()); - for (IDPredicate predicate : predicates) { + for (TagPredicate predicate : predicates) { serializedSize += predicate.serializedSize(); } return serializedSize; @@ -370,7 +380,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public long serialize(OutputStream stream) throws IOException { long size = super.serialize(stream); size += ReadWriteForEncodingUtils.writeVarInt(predicates.size(), stream); - for (IDPredicate predicate : predicates) { + for (TagPredicate predicate : predicates) { size += predicate.serialize(stream); } return size; @@ -380,7 +390,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public long serialize(ByteBuffer buffer) { long size = super.serialize(buffer); size += ReadWriteForEncodingUtils.writeVarInt(predicates.size(), buffer); - for (IDPredicate predicate : predicates) { + for (TagPredicate predicate : predicates) { size += predicate.serialize(buffer); } return size; @@ -390,7 +400,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public void deserialize(InputStream stream) throws IOException { int size = ReadWriteForEncodingUtils.readVarInt(stream); for (int i = 0; i < size; i++) { - predicates.add(IDPredicate.createFrom(stream)); + predicates.add(TagPredicate.createFrom(stream)); } } @@ -398,7 +408,7 @@ public abstract class IDPredicate implements StreamSerializable, BufferSerializa public void deserialize(ByteBuffer buffer) { int size = ReadWriteForEncodingUtils.readVarInt(buffer); for (int i = 0; i < size; i++) { - predicates.add(IDPredicate.createFrom(buffer)); + predicates.add(TagPredicate.createFrom(buffer)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java index 9a8d2cda88a..6e50bf5f0c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/evolution/EvolvedSchema.java @@ -21,6 +21,12 @@ package org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry.ModType; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; @@ -40,7 +46,7 @@ public class EvolvedSchema { private Map<String, Map<String, String>> originalColumnNames = new LinkedHashMap<>(); public void renameTable(String oldTableName, String newTableName) { - if (!originalTableNames.containsKey(oldTableName)) { + if (!originalTableNames.containsKey(oldTableName) || originalTableNames.get(oldTableName).isEmpty()) { originalTableNames.put(newTableName, oldTableName); originalTableNames.put(oldTableName, ""); } else { @@ -58,7 +64,7 @@ public class EvolvedSchema { public void renameColumn(String tableName, String oldColumnName, String newColumnName) { Map<String, String> columnNameMap = originalColumnNames.computeIfAbsent(tableName, t -> new LinkedHashMap<>()); - if (!columnNameMap.containsKey(oldColumnName)) { + if (!columnNameMap.containsKey(oldColumnName) || columnNameMap.get(oldColumnName).isEmpty()) { columnNameMap.put(newColumnName, oldColumnName); columnNameMap.put(oldColumnName, ""); } else { @@ -122,14 +128,36 @@ public class EvolvedSchema { return schemaEvolutions; } - public IDeviceID rewriteDeviceId(IDeviceID deviceID) { + public ModEntry rewriteToOriginal(ModEntry entry) { + if (entry.getType() == ModType.TABLE_DELETION) { + return rewriteToOriginal(((TableDeletionEntry) entry)); + } + return entry; + } + + public TableDeletionEntry rewriteToOriginal(TableDeletionEntry entry) { + DeletionPredicate deletionPredicate = rewriteToOriginal(entry.getPredicate()); + return new TableDeletionEntry(deletionPredicate, entry.getTimeRange()); + } + + private DeletionPredicate rewriteToOriginal(DeletionPredicate predicate) { + String originalTableName = getOriginalTableName(predicate.getTableName()); + TagPredicate tagPredicate = predicate.getTagPredicate(); + tagPredicate = tagPredicate.rewriteToOriginal(this); + List<String> newMeasurements = + predicate.getMeasurementNames().stream().map(m -> getOriginalColumnName(predicate.getTableName(), m)).collect( + Collectors.toList()); + return new DeletionPredicate(originalTableName, tagPredicate, newMeasurements); + } + + public IDeviceID rewriteToOriginal(IDeviceID deviceID) { String tableName = deviceID.getTableName(); String originalTableName = getOriginalTableName(tableName); - return rewriteDeviceId(deviceID, originalTableName); + return rewriteToOriginal(deviceID, originalTableName); } @SuppressWarnings("SuspiciousSystemArraycopy") - public static IDeviceID rewriteDeviceId(IDeviceID deviceID, String originalTableName) { + public static IDeviceID rewriteToOriginal(IDeviceID deviceID, String originalTableName) { String tableName = deviceID.getTableName(); if (!tableName.equals(originalTableName)) { Object[] segments = deviceID.getSegments(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java index aaa7bb195ca..b9fd86316b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/fileset/TsFileSet.java @@ -62,7 +62,7 @@ public class TsFileSet implements Comparable<TsFileSet> { } if (schemaEvolutionFile == null) { - schemaEvolutionFile = new SchemaEvolutionFile(0 + SchemaEvolutionFile.FILE_SUFFIX); + schemaEvolutionFile = new SchemaEvolutionFile(fileSetsDir + File.separator + 0 + SchemaEvolutionFile.FILE_SUFFIX); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index f90cb0353bb..6569a7c6eb3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -34,7 +34,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.IWritableMemChunkGroup; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java index 7a7d71bef51..1db3890d9fe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/path/PatternTreeMapTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PatternTreeMap; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java index 06f823c0e23..6eeeda629eb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionRecoverTest.java @@ -30,7 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDele import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.tsfile.read.common.TimeRange; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java index f94d909f94b..bc6dc9e625a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/consensus/DeletionResourceTest.java @@ -43,7 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.AbstractDele import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java index d01351ce60a..63909828e6a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/PipePlanTablePatternParseVisitorTest.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.Creat import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.storageengine.dataregion.memtable.DeviceIDFactory; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.tsfile.read.common.TimeRange; @@ -115,12 +115,12 @@ public class PipePlanTablePatternParseVisitorTest { new TableDeletionEntry( new DeletionPredicate( "ac", - new IDPredicate.And( - new IDPredicate.FullExactMatch( + new TagPredicate.And( + new TagPredicate.FullExactMatch( DeviceIDFactory.getInstance() .getDeviceID( new PartialPath(new String[] {"ac", "device1"}))), - new IDPredicate.SegmentExactMatch("device2", 1))), + new TagPredicate.SegmentExactMatch("device2", 1))), new TimeRange(0, 1))), "db1"), tablePattern) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java index e51a8b99db9..a6b6bf1bf67 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalDeleteDataNodeTest.java @@ -24,10 +24,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.And; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALByteBufferForTest; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java index 014874e5893..ec9a4f5eae2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/DataRegionTest.java @@ -45,6 +45,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.DeleteDataNo import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils; @@ -60,10 +61,18 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.constant import org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer; import org.apache.iotdb.db.storageengine.dataregion.flush.FlushManager; import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy; +import org.apache.iotdb.db.storageengine.dataregion.flush.TsFileFlushPolicy.DirectFlushPolicy; import org.apache.iotdb.db.storageengine.dataregion.memtable.ReadOnlyMemChunk; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; +import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource.ModIterator; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.ColumnRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.evolution.TableRename; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; import org.apache.iotdb.db.storageengine.rescon.memory.MemTableManager; @@ -77,6 +86,7 @@ import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TimeValuePair; +import org.apache.tsfile.read.common.TimeRange; import org.apache.tsfile.read.reader.IPointReader; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.BitMap; @@ -102,6 +112,7 @@ import java.util.List; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertRowNode; import static org.apache.iotdb.db.queryengine.plan.statement.StatementTestUtils.genInsertTabletNode; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class DataRegionTest { @@ -113,7 +124,7 @@ public class DataRegionTest { private String systemDir = TestConstant.OUTPUT_DATA_DIR.concat("info"); private String deviceId = "root.vehicle.d0"; - private IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId); + private IDeviceID device = Factory.DEFAULT_FACTORY.create(deviceId); private String measurementId = "s0"; private NonAlignedFullPath nonAlignedFullPath = @@ -242,7 +253,7 @@ public class DataRegionTest { null); } - Assert.assertEquals(1, tsfileResourcesForQuery.size()); + assertEquals(1, tsfileResourcesForQuery.size()); List<ReadOnlyMemChunk> memChunks = tsfileResourcesForQuery.get(0).getReadOnlyMemChunk(IFullPath.convertToIFullPath(fullPath)); long time = 16; @@ -250,7 +261,7 @@ public class DataRegionTest { IPointReader iterator = memChunk.getPointReader(); while (iterator.hasNextTimeValuePair()) { TimeValuePair timeValuePair = iterator.nextTimeValuePair(); - Assert.assertEquals(time++, timeValuePair.getTimestamp()); + assertEquals(time++, timeValuePair.getTimestamp()); } } } @@ -265,7 +276,7 @@ public class DataRegionTest { dataRegion.syncCloseAllWorkingTsFileProcessors(); } - IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create(deviceId); + IDeviceID device = Factory.DEFAULT_FACTORY.create(deviceId); QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList( @@ -275,7 +286,7 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(10, queryDataSource.getSeqResources().size()); + assertEquals(10, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -308,8 +319,8 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); queryDataSource = dataRegion.query( @@ -322,8 +333,8 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -356,8 +367,8 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); queryDataSource = dataRegion.query( @@ -370,8 +381,8 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -429,8 +440,8 @@ public class DataRegionTest { int hashCode2 = Arrays.hashCode((long[]) columns[1]); dataRegion.insertTablet(insertTabletNode1); // the hashCode should not be changed when insert - Assert.assertEquals(hashCode1, Arrays.hashCode((int[]) columns[0])); - Assert.assertEquals(hashCode2, Arrays.hashCode((long[]) columns[1])); + assertEquals(hashCode1, Arrays.hashCode((int[]) columns[0])); + assertEquals(hashCode2, Arrays.hashCode((long[]) columns[1])); dataRegion.syncCloseAllWorkingTsFileProcessors(); for (int r = 50; r < 149; r++) { @@ -459,8 +470,8 @@ public class DataRegionTest { dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(2, queryDataSource.getSeqResources().size()); - Assert.assertEquals(1, queryDataSource.getUnseqResources().size()); + assertEquals(2, queryDataSource.getSeqResources().size()); + assertEquals(1, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -531,14 +542,14 @@ public class DataRegionTest { dataRegion.insertTablet(insertTabletNode2); assertTrue(SystemInfo.getInstance().getTotalMemTableSize() > 0); dataRegion.syncDeleteDataFiles(); - Assert.assertEquals(0, SystemInfo.getInstance().getTotalMemTableSize()); + assertEquals(0, SystemInfo.getInstance().getTotalMemTableSize()); QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); } @Test @@ -611,8 +622,8 @@ public class DataRegionTest { dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -687,8 +698,8 @@ public class DataRegionTest { dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -714,8 +725,8 @@ public class DataRegionTest { QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(10, queryDataSource.getSeqResources().size()); - Assert.assertEquals(10, queryDataSource.getUnseqResources().size()); + assertEquals(10, queryDataSource.getSeqResources().size()); + assertEquals(10, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -748,8 +759,8 @@ public class DataRegionTest { QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -781,8 +792,8 @@ public class DataRegionTest { QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(20, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(20, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -863,8 +874,8 @@ public class DataRegionTest { dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -943,8 +954,8 @@ public class DataRegionTest { dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1023,8 +1034,8 @@ public class DataRegionTest { dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(0, queryDataSource.getSeqResources().size()); - Assert.assertEquals(2, queryDataSource.getUnseqResources().size()); + assertEquals(0, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1053,7 +1064,7 @@ public class DataRegionTest { InsertRowsNode insertRowsNode = new InsertRowsNode(new PlanNodeId(""), indexList, nodes); dataRegion1.insert(insertRowsNode); dataRegion1.syncCloseAllWorkingTsFileProcessors(); - IDeviceID tmpDeviceId = IDeviceID.Factory.DEFAULT_FACTORY.create("root.Rows"); + IDeviceID tmpDeviceId = Factory.DEFAULT_FACTORY.create("root.Rows"); QueryDataSource queryDataSource = dataRegion1.query( Collections.singletonList( @@ -1063,8 +1074,8 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(1, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(1, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1090,7 +1101,7 @@ public class DataRegionTest { dataRegion1.syncCloseAllWorkingTsFileProcessors(); } dataRegion1.syncCloseAllWorkingTsFileProcessors(); - IDeviceID tmpDeviceId = IDeviceID.Factory.DEFAULT_FACTORY.create("root.ln22"); + IDeviceID tmpDeviceId = Factory.DEFAULT_FACTORY.create("root.ln22"); QueryDataSource queryDataSource = dataRegion1.query( Collections.singletonList( @@ -1100,8 +1111,8 @@ public class DataRegionTest { context, null, null); - Assert.assertEquals(10, queryDataSource.getSeqResources().size()); - Assert.assertEquals(0, queryDataSource.getUnseqResources().size()); + assertEquals(10, queryDataSource.getSeqResources().size()); + assertEquals(0, queryDataSource.getUnseqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1169,7 +1180,7 @@ public class DataRegionTest { QueryDataSource queryDataSource = dataRegion.query( Collections.singletonList(nonAlignedFullPath), device, context, null, null); - Assert.assertEquals(2, queryDataSource.getSeqResources().size()); + assertEquals(2, queryDataSource.getSeqResources().size()); for (TsFileResource resource : queryDataSource.getSeqResources()) { assertTrue(resource.isClosed()); } @@ -1256,7 +1267,7 @@ public class DataRegionTest { TSRecord record = new TSRecord(deviceId, 10000); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1000))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); - Assert.assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); // change config & reboot timed service boolean prevEnableTimedFlushSeqMemtable = config.isEnableTimedFlushSeqMemtable(); @@ -1267,7 +1278,7 @@ public class DataRegionTest { Thread.sleep(500); - Assert.assertEquals(1, dataRegion.getWorkSequenceTsFileProcessors().size()); + assertEquals(1, dataRegion.getWorkSequenceTsFileProcessors().size()); TsFileProcessor tsFileProcessor = dataRegion.getWorkSequenceTsFileProcessors().iterator().next(); FlushManager flushManager = FlushManager.getInstance(); @@ -1292,7 +1303,7 @@ public class DataRegionTest { } } - Assert.assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); config.setEnableTimedFlushSeqMemtable(prevEnableTimedFlushSeqMemtable); config.setSeqMemtableFlushInterval(preFLushInterval); @@ -1305,15 +1316,15 @@ public class DataRegionTest { TSRecord record = new TSRecord(deviceId, 10000); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1000))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); - Assert.assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); dataRegion.syncCloseAllWorkingTsFileProcessors(); - Assert.assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); // create one unsequence memtable record = new TSRecord(deviceId, 1); record.addTuple(DataPoint.getDataPoint(TSDataType.INT32, measurementId, String.valueOf(1000))); dataRegion.insert(buildInsertRowNodeByTSRecord(record)); - Assert.assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(1, MemTableManager.getInstance().getCurrentMemtableNumber()); // change config & reboot timed service boolean prevEnableTimedFlushUnseqMemtable = config.isEnableTimedFlushUnseqMemtable(); @@ -1324,7 +1335,7 @@ public class DataRegionTest { Thread.sleep(500); - Assert.assertEquals(1, dataRegion.getWorkUnsequenceTsFileProcessors().size()); + assertEquals(1, dataRegion.getWorkUnsequenceTsFileProcessors().size()); TsFileProcessor tsFileProcessor = dataRegion.getWorkUnsequenceTsFileProcessors().iterator().next(); FlushManager flushManager = FlushManager.getInstance(); @@ -1349,7 +1360,7 @@ public class DataRegionTest { } } - Assert.assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); + assertEquals(0, MemTableManager.getInstance().getCurrentMemtableNumber()); config.setEnableTimedFlushUnseqMemtable(prevEnableTimedFlushUnseqMemtable); config.setUnseqMemtableFlushInterval(preFLushInterval); @@ -1404,10 +1415,10 @@ public class DataRegionTest { TsFileResource resource = dataRegion.getSequenceFileList().get(i); if (i == 1) { assertTrue(resource.anyModFileExists()); - Assert.assertEquals(2, resource.getAllModEntries().size()); + assertEquals(2, resource.getAllModEntries().size()); } else if (i == 3) { assertTrue(resource.anyModFileExists()); - Assert.assertEquals(1, resource.getAllModEntries().size()); + assertEquals(1, resource.getAllModEntries().size()); } else { Assert.assertFalse(resource.anyModFileExists()); } @@ -1501,7 +1512,7 @@ public class DataRegionTest { dataRegion.syncCloseAllWorkingTsFileProcessors(); assertTrue(tsFileResource.anyModFileExists()); - Assert.assertEquals(3, tsFileResource.getAllModEntries().size()); + assertEquals(3, tsFileResource.getAllModEntries().size()); } @Test @@ -1596,7 +1607,7 @@ public class DataRegionTest { dataRegion.syncCloseAllWorkingTsFileProcessors(); assertTrue(tsFileResource.anyModFileExists()); - Assert.assertEquals(3, tsFileResource.getAllModEntries().size()); + assertEquals(3, tsFileResource.getAllModEntries().size()); } @Test @@ -1638,7 +1649,7 @@ public class DataRegionTest { Assert.assertFalse( tsFileResource .getDevices() - .contains(IDeviceID.Factory.DEFAULT_FACTORY.create("root.vehicle.d199"))); + .contains(Factory.DEFAULT_FACTORY.create("root.vehicle.d199"))); } @Test @@ -1670,7 +1681,7 @@ public class DataRegionTest { public DummyDataRegion(String systemInfoDir, String storageGroupName) throws DataRegionException { - super(systemInfoDir, "0", new TsFileFlushPolicy.DirectFlushPolicy(), storageGroupName); + super(systemInfoDir, "0", new DirectFlushPolicy(), storageGroupName); } } @@ -1786,7 +1797,7 @@ public class DataRegionTest { @Test public void testSchemaEvolution() - throws IllegalPathException, WriteProcessException, QueryProcessException, IOException { + throws WriteProcessException, QueryProcessException, IOException { String[] measurements = {"tag1", "s1", "s2"}; MeasurementSchema[] measurementSchemas = { new MeasurementSchema("tag1", TSDataType.STRING), @@ -1896,4 +1907,120 @@ public class DataRegionTest { Long.MAX_VALUE); assertEquals(1, dataSource.getSeqResources().size()); } + + @Test + public void testSchemaEvolutionWithPartialDeletion() + throws WriteProcessException, IOException { + String[] measurements = {"tag1", "s1", "s2"}; + MeasurementSchema[] measurementSchemas = { + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }; + RelationalInsertRowNode insertRowNode = + new RelationalInsertRowNode( + new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[] {TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + }); + dataRegion.insert(insertRowNode); + insertRowNode.setTime(20); + dataRegion.insert(insertRowNode); + + // table1 -> table2 + dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); + // s1 -> s3 + dataRegion.applySchemaEvolution(Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); + + // delete with table2 + TableDeletionEntry tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 15)); + RelationalDeleteDataNode relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // delete with s3 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2", new NOP(), Collections.singletonList("s3")), new TimeRange(0, 15)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // delete with table1 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 15)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // delete with s1 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2", new NOP(), Collections.singletonList("s1")), new TimeRange(0, 15)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + + List<TsFileResource> sequenceFileList = dataRegion.getSequenceFileList(); + assertEquals(1, sequenceFileList.size()); + ModIterator modEntryIterator = sequenceFileList.get(0).getModEntryIterator(); + ModEntry next = modEntryIterator.next(); + // the table2 modification should be rewritten to table1 + assertEquals("table1", ((TableDeletionEntry) next).getTableName()); + next = modEntryIterator.next(); + // the s3 modification should be rewritten to s1 + assertEquals(Collections.singletonList("s1"), ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); + next = modEntryIterator.next(); + // the table1 modification should be skipped + // the s1 modification should be rewritten to empty + assertEquals(Collections.singletonList(""), ((TableDeletionEntry) next).getPredicate().getMeasurementNames()); + assertFalse(modEntryIterator.hasNext()); + } + + @Test + public void testSchemaEvolutionWithFullDeletion() + throws WriteProcessException, IOException { + String[] measurements = {"tag1", "s1", "s2"}; + MeasurementSchema[] measurementSchemas = { + new MeasurementSchema("tag1", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT64), + new MeasurementSchema("s2", TSDataType.DOUBLE) + }; + RelationalInsertRowNode insertRowNode = + new RelationalInsertRowNode( + new PlanNodeId(""), + new PartialPath(new String[] {"table1"}), + true, + measurements, + new TSDataType[] {TSDataType.STRING, TSDataType.INT64, TSDataType.DOUBLE}, + measurementSchemas, + 10, + new Object[] {new Binary("tag1".getBytes(StandardCharsets.UTF_8)), 1L, 1.0}, + false, + new TsTableColumnCategory[] { + TsTableColumnCategory.TAG, TsTableColumnCategory.FIELD, TsTableColumnCategory.FIELD + }); + dataRegion.insert(insertRowNode); + insertRowNode.setTime(20); + dataRegion.insert(insertRowNode); + + // table1 -> table2 + dataRegion.applySchemaEvolution(Collections.singletonList(new TableRename("table1", "table2"))); + // s1 -> s3 + dataRegion.applySchemaEvolution(Collections.singletonList(new ColumnRename("table2", "s1", "s3", null))); + + // delete with table1 + TableDeletionEntry tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table1"), new TimeRange(0, 30)); + RelationalDeleteDataNode relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // nothing should be deleted + List<TsFileResource> sequenceFileList = dataRegion.getSequenceFileList(); + assertEquals(1, sequenceFileList.size()); + ModIterator modEntryIterator = sequenceFileList.get(0).getModEntryIterator(); + assertFalse(modEntryIterator.hasNext()); + + // delete with table2 + tableDeletionEntry = new TableDeletionEntry(new DeletionPredicate("table2"), new TimeRange(0, 30)); + relationalDeleteDataNode = new RelationalDeleteDataNode(new PlanNodeId(""), tableDeletionEntry, dataRegion.getDatabaseName()); + dataRegion.deleteByTable(relationalDeleteDataNode); + // the file should be deleted + sequenceFileList = dataRegion.getSequenceFileList(); + assertEquals(0, sequenceFileList.size()); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index 9ff8a401150..a83fa920853 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -35,8 +35,8 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.modification.DeletionPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; @@ -333,7 +333,7 @@ public class CompactionWithAllNullRowsTest extends AbstractCompactionTest { new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s0")), new TimeRange(Long.MIN_VALUE, 11))); resource1 @@ -342,7 +342,7 @@ public class CompactionWithAllNullRowsTest extends AbstractCompactionTest { new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s1")), new TimeRange(Long.MIN_VALUE, 11))); resource1 @@ -351,7 +351,7 @@ public class CompactionWithAllNullRowsTest extends AbstractCompactionTest { new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s2")), new TimeRange(Long.MIN_VALUE, 11))); resource1 @@ -360,7 +360,7 @@ public class CompactionWithAllNullRowsTest extends AbstractCompactionTest { new TableDeletionEntry( new DeletionPredicate( "t1", - new IDPredicate.FullExactMatch(deviceID), + new TagPredicate.FullExactMatch(deviceID), Collections.singletonList("s3")), new TimeRange(Long.MIN_VALUE, 11))); resource1.getModFileForWrite().close(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java index a0a9885ecf0..29609cc9c27 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java @@ -22,9 +22,9 @@ package org.apache.iotdb.db.storageengine.dataregion.modification; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover.CompactionRecoverManager; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.file.metadata.IDeviceID.Factory; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java index 5c2979a9075..ee9a4dfa405 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/TableDeletionEntryTest.java @@ -18,10 +18,10 @@ */ package org.apache.iotdb.db.storageengine.dataregion.modification; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.And; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.FullExactMatch; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.NOP; -import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate.SegmentExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.And; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.FullExactMatch; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.NOP; +import org.apache.iotdb.db.storageengine.dataregion.modification.TagPredicate.SegmentExactMatch; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory;
