This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 36c7db80d15 fix redundant operations in compaction (#17239)
36c7db80d15 is described below
commit 36c7db80d15339443dd50a292e69c0f9a69145ed
Author: shuwenwei <[email protected]>
AuthorDate: Mon Mar 2 16:23:36 2026 +0800
fix redundant operations in compaction (#17239)
---
.../performer/impl/FastCompactionPerformer.java | 7 --
.../execute/utils/MultiTsFileDeviceIterator.java | 93 ++++------------------
.../fast/FastAlignedSeriesCompactionExecutor.java | 18 +++--
3 files changed, 26 insertions(+), 92 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
index 54b21ddd382..629493f184b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
@@ -193,13 +193,6 @@ public class FastCompactionPerformer
ttlDeletion =
CompactionUtils.convertTtlToDeletion(
device, deviceIterator.getTimeLowerBoundForCurrentDevice());
- for (TsFileResource sourceFile : sortedSourceFiles) {
- modificationCache
- .computeIfAbsent(
- sourceFile.getTsFile().getName(),
- k -> PatternTreeMapFactory.getModsPatternTreeMap())
- .append(ttlDeletion.keyOfPatternTree(), ttlDeletion);
- }
}
compactionWriter.setTTLDeletion(ttlDeletion);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index 9099586e87e..31296771090 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -294,79 +294,9 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
* Get all measurements and their timeseries metadata offset in each source
file. It is used for
* new fast compaction to compact nonAligned timeseries.
*
- * @return measurement -> tsfile resource -> timeseries metadata
<startOffset, endOffset>
+ * @return measurement -> CompactionSeriesContext
* @throws IOException if io errors occurred
*/
- public Map<String, Map<TsFileResource, Pair<Long, Long>>>
- getTimeseriesMetadataOffsetOfCurrentDevice() throws IOException {
- Map<String, Map<TsFileResource, Pair<Long, Long>>>
timeseriesMetadataOffsetMap =
- new HashMap<>();
- Map<String, TSDataType> measurementDataTypeMap = new HashMap<>();
- for (TsFileResource resource : tsFileResourcesSortedByDesc) {
- if (!deviceIteratorMap.containsKey(resource)
- || !deviceIteratorMap.get(resource).current().equals(currentDevice))
{
- // if this tsfile has no more device or next device is not equals to
the current device,
- // which means this tsfile does not contain the current device, then
skip it.
- continue;
- }
- TsFileSequenceReader reader = readerMap.get(resource);
- for (Map.Entry<String, Pair<TimeseriesMetadata, Pair<Long, Long>>>
entrySet :
- ((CompactionTsFileReader) reader)
- .getTimeseriesMetadataAndOffsetByDevice(
-
deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(),
- Collections.emptySet(),
- false)
- .entrySet()) {
- String measurementId = entrySet.getKey();
- // skip the TimeseriesMetadata whose data type is not consistent
- TSDataType dataTypeOfCurrentTimeseriesMetadata =
entrySet.getValue().left.getTsDataType();
- TSDataType correctDataTypeOfCurrentMeasurement =
- measurementDataTypeMap.putIfAbsent(measurementId,
dataTypeOfCurrentTimeseriesMetadata);
- if (correctDataTypeOfCurrentMeasurement != null
- && !MetadataUtils.canAlter(
- dataTypeOfCurrentTimeseriesMetadata,
correctDataTypeOfCurrentMeasurement)) {
- continue;
- }
- timeseriesMetadataOffsetMap.putIfAbsent(measurementId, new
HashMap<>());
- timeseriesMetadataOffsetMap.get(measurementId).put(resource,
entrySet.getValue().right);
- }
- }
- return timeseriesMetadataOffsetMap;
- }
-
- /**
- * Get all measurement data types of the current device from source files.
Traverse all the files
- * from the newest to the oldest in turn and start traversing the index tree
from the
- * firstMeasurementNode node to get all the measurement types under the
current device.
- *
- * @return measurement -> data type
- * @throws IOException if io errors occurred
- */
- public Map<String, TSDataType> getDataTypeOfCurrentDevice() throws
IOException {
- Map<String, TSDataType> measurementNameDataTypeMap = new HashMap<>();
- for (TsFileResource resource : tsFileResourcesSortedByDesc) {
- if (!deviceIteratorMap.containsKey(resource)
- || !deviceIteratorMap.get(resource).current().equals(currentDevice))
{
- // if this tsfile has no more device or next device is not equals to
the current device,
- // which means this tsfile does not contain the current device, then
skip it.
- continue;
- }
- TsFileSequenceReader reader = readerMap.get(resource);
- for (Map.Entry<String, Pair<TimeseriesMetadata, Pair<Long, Long>>>
entrySet :
- ((CompactionTsFileReader) reader)
- .getTimeseriesMetadataAndOffsetByDevice(
-
deviceIteratorMap.get(resource).getFirstMeasurementNodeOfCurrentDevice(),
- Collections.emptySet(),
- false)
- .entrySet()) {
- String measurementId = entrySet.getKey();
- TSDataType dataType = entrySet.getValue().left.getTsDataType();
- measurementNameDataTypeMap.putIfAbsent(measurementId, dataType);
- }
- }
- return measurementNameDataTypeMap;
- }
-
public Map<String, CompactionSeriesContext>
getCompactionSeriesContextOfCurrentDevice()
throws IOException {
Map<String, CompactionSeriesContext> compactionSeriesContextMap = new
HashMap<>();
@@ -388,16 +318,21 @@ public class MultiTsFileDeviceIterator implements
AutoCloseable {
String measurementId = entrySet.getKey();
TimeseriesMetadata timeseriesMetadata = entrySet.getValue().left;
Pair<Long, Long> offset = entrySet.getValue().right;
- TSDataType dataType = entrySet.getValue().left.getTsDataType();
- if (compactionSeriesContextMap.get(measurementId) != null
- && compactionSeriesContextMap.get(measurementId).getFinalType() !=
null
- && !MetadataUtils.canAlter(
- dataType,
compactionSeriesContextMap.get(measurementId).getFinalType())) {
+ TSDataType dataType = timeseriesMetadata.getTsDataType();
+
+ CompactionSeriesContext compactionSeriesContext =
+ compactionSeriesContextMap.get(measurementId);
+ if (compactionSeriesContext != null
+ && compactionSeriesContext.getFinalType() != null
+ && !MetadataUtils.canAlter(dataType,
compactionSeriesContext.getFinalType())) {
continue;
}
- compactionSeriesContextMap.putIfAbsent(measurementId, new
CompactionSeriesContext());
- compactionSeriesContextMap.get(measurementId).put(resource,
entrySet.getValue().right);
-
compactionSeriesContextMap.get(measurementId).setFinalTypeIfAbsent(dataType);
+
+ compactionSeriesContext =
+ compactionSeriesContextMap.computeIfAbsent(
+ measurementId, k -> new CompactionSeriesContext());
+ compactionSeriesContext.put(resource, offset);
+ compactionSeriesContext.setFinalTypeIfAbsent(dataType);
}
}
return compactionSeriesContextMap;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
index 121d4ca1d3a..cf9ed65b819 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
@@ -276,21 +276,27 @@ public class FastAlignedSeriesCompactionExecutor extends
SeriesCompactionExecuto
private boolean isValueChunkDataTypeMatchSchema(
List<IChunkMetadata> chunkMetadataListOfOneValueColumn) {
- boolean isMatch = false;
+ boolean needAlter = false;
for (IChunkMetadata chunkMetadata : chunkMetadataListOfOneValueColumn) {
if (chunkMetadata == null) {
continue;
}
String measurement = chunkMetadata.getMeasurementUid();
IMeasurementSchema schema = measurementSchemaMap.get(measurement);
- if (MetadataUtils.canAlter(chunkMetadata.getDataType(),
schema.getType())) {
- if (schema.getType() != chunkMetadata.getDataType()) {
- chunkMetadata.setNewType(schema.getType());
+ if (!needAlter) {
+ // Since all chunks in chunkMetadataListOfOneValueColumn share the
same dataType, perform
+ // the compatibility check and early-return only on the first non-null
chunk.
+ if (!MetadataUtils.canAlter(chunkMetadata.getDataType(),
schema.getType())) {
+ return false;
+ }
+ if (schema.getType() == chunkMetadata.getDataType()) {
+ return true;
}
- isMatch = true;
}
+ needAlter = true;
+ chunkMetadata.setNewType(schema.getType());
}
- return isMatch;
+ return true;
}
/**
