This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fixRedundantOperationsInCompaction in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2cbd4f821d9ec548931ff54052a692ac097b796b Author: shuwenwei <[email protected]> AuthorDate: Mon Mar 2 11:47:28 2026 +0800 fix redundant operations in compaction --- .../performer/impl/FastCompactionPerformer.java | 7 -- .../execute/utils/MultiTsFileDeviceIterator.java | 93 ++++------------------ .../fast/FastAlignedSeriesCompactionExecutor.java | 18 +++-- 3 files changed, 26 insertions(+), 92 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java index 54b21ddd382..629493f184b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java @@ -193,13 +193,6 @@ public class FastCompactionPerformer ttlDeletion = CompactionUtils.convertTtlToDeletion( device, deviceIterator.getTimeLowerBoundForCurrentDevice()); - for (TsFileResource sourceFile : sortedSourceFiles) { - modificationCache - .computeIfAbsent( - sourceFile.getTsFile().getName(), - k -> PatternTreeMapFactory.getModsPatternTreeMap()) - .append(ttlDeletion.keyOfPatternTree(), ttlDeletion); - } } compactionWriter.setTTLDeletion(ttlDeletion); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java index 9099586e87e..31296771090 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java @@ -294,79 +294,9 @@ public class MultiTsFileDeviceIterator implements AutoCloseable { * Get all measurements and their timeseries metadata offset in each source file. It is used for * new fast compaction to compact nonAligned timeseries. * - * @return measurement -> tsfile resource -> timeseries metadata <startOffset, endOffset> + * @return measurement -> CompactionSeriesContext * @throws IOException if io errors occurred */ - public Map<String, Map<TsFileResource, Pair<Long, Long>>> - getTimeseriesMetadataOffsetOfCurrentDevice() throws IOException { - Map<String, Map<TsFileResource, Pair<Long, Long>>> timeseriesMetadataOffsetMap = - new HashMap<>(); - Map<String, TSDataType> measurementDataTypeMap = new HashMap<>(); - for (TsFileResource resource : tsFileResourcesSortedByDesc) { - if (!deviceIteratorMap.containsKey(resource) - || !deviceIteratorMap.get(resource).current().equals(currentDevice)) { - // if this tsfile has no more device or next device is not equals to the current device, - // which means this tsfile does not contain the current device, then skip it. - continue; - } - TsFileSequenceReader reader = readerMap.get(resource); - for (Map.Entry<String, Pair<TimeseriesMetadata, Pair<Long, Long>>> entrySet : - ((CompactionTsFileReader) reader) - .getTimeseriesMetadataAndOffsetByDevice( - deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), - Collections.emptySet(), - false) - .entrySet()) { - String measurementId = entrySet.getKey(); - // skip the TimeseriesMetadata whose data type is not consistent - TSDataType dataTypeOfCurrentTimeseriesMetadata = entrySet.getValue().left.getTsDataType(); - TSDataType correctDataTypeOfCurrentMeasurement = - measurementDataTypeMap.putIfAbsent(measurementId, dataTypeOfCurrentTimeseriesMetadata); - if (correctDataTypeOfCurrentMeasurement != null - && !MetadataUtils.canAlter( - dataTypeOfCurrentTimeseriesMetadata, correctDataTypeOfCurrentMeasurement)) { - continue; - } - timeseriesMetadataOffsetMap.putIfAbsent(measurementId, new HashMap<>()); - timeseriesMetadataOffsetMap.get(measurementId).put(resource, entrySet.getValue().right); - } - } - return timeseriesMetadataOffsetMap; - } - - /** - * Get all measurement data types of the current device from source files. Traverse all the files - * from the newest to the oldest in turn and start traversing the index tree from the - * firstMeasurementNode node to get all the measurement types under the current device. - * - * @return measurement -> data type - * @throws IOException if io errors occurred - */ - public Map<String, TSDataType> getDataTypeOfCurrentDevice() throws IOException { - Map<String, TSDataType> measurementNameDataTypeMap = new HashMap<>(); - for (TsFileResource resource : tsFileResourcesSortedByDesc) { - if (!deviceIteratorMap.containsKey(resource) - || !deviceIteratorMap.get(resource).current().equals(currentDevice)) { - // if this tsfile has no more device or next device is not equals to the current device, - // which means this tsfile does not contain the current device, then skip it. - continue; - } - TsFileSequenceReader reader = readerMap.get(resource); - for (Map.Entry<String, Pair<TimeseriesMetadata, Pair<Long, Long>>> entrySet : - ((CompactionTsFileReader) reader) - .getTimeseriesMetadataAndOffsetByDevice( - deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(), - Collections.emptySet(), - false) - .entrySet()) { - String measurementId = entrySet.getKey(); - TSDataType dataType = entrySet.getValue().left.getTsDataType(); - measurementNameDataTypeMap.putIfAbsent(measurementId, dataType); - } - } - return measurementNameDataTypeMap; - } - public Map<String, CompactionSeriesContext> getCompactionSeriesContextOfCurrentDevice() throws IOException { Map<String, CompactionSeriesContext> compactionSeriesContextMap = new HashMap<>(); @@ -388,16 +318,21 @@ public class MultiTsFileDeviceIterator implements AutoCloseable { String measurementId = entrySet.getKey(); TimeseriesMetadata timeseriesMetadata = entrySet.getValue().left; Pair<Long, Long> offset = entrySet.getValue().right; - TSDataType dataType = entrySet.getValue().left.getTsDataType(); - if (compactionSeriesContextMap.get(measurementId) != null - && compactionSeriesContextMap.get(measurementId).getFinalType() != null - && !MetadataUtils.canAlter( - dataType, compactionSeriesContextMap.get(measurementId).getFinalType())) { + TSDataType dataType = timeseriesMetadata.getTsDataType(); + + CompactionSeriesContext compactionSeriesContext = + compactionSeriesContextMap.get(measurementId); + if (compactionSeriesContext != null + && compactionSeriesContext.getFinalType() != null + && !MetadataUtils.canAlter(dataType, compactionSeriesContext.getFinalType())) { continue; } - compactionSeriesContextMap.putIfAbsent(measurementId, new CompactionSeriesContext()); - compactionSeriesContextMap.get(measurementId).put(resource, entrySet.getValue().right); - compactionSeriesContextMap.get(measurementId).setFinalTypeIfAbsent(dataType); + + compactionSeriesContext = + compactionSeriesContextMap.computeIfAbsent( + measurementId, k -> new CompactionSeriesContext()); + compactionSeriesContext.put(resource, offset); + compactionSeriesContext.setFinalTypeIfAbsent(dataType); } } return compactionSeriesContextMap; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java index 121d4ca1d3a..cf9ed65b819 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java @@ -276,21 +276,27 @@ public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecuto private boolean isValueChunkDataTypeMatchSchema( List<IChunkMetadata> chunkMetadataListOfOneValueColumn) { - boolean isMatch = false; + boolean needAlter = false; for (IChunkMetadata chunkMetadata : chunkMetadataListOfOneValueColumn) { if (chunkMetadata == null) { continue; } String measurement = chunkMetadata.getMeasurementUid(); IMeasurementSchema schema = measurementSchemaMap.get(measurement); - if (MetadataUtils.canAlter(chunkMetadata.getDataType(), schema.getType())) { - if (schema.getType() != chunkMetadata.getDataType()) { - chunkMetadata.setNewType(schema.getType()); + if (!needAlter) { + // Since all chunks in chunkMetadataListOfOneValueColumn share the same dataType, perform + // the compatibility check and early-return only on the first non-null chunk. + if (!MetadataUtils.canAlter(chunkMetadata.getDataType(), schema.getType())) { + return false; + } + if (schema.getType() == chunkMetadata.getDataType()) { + return true; } - isMatch = true; } + needAlter = true; + chunkMetadata.setNewType(schema.getType()); } - return isMatch; + return true; } /**
