This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d74fca109dc Repair the problem that accumulate index used
chunkMetadataList (#17040)
d74fca109dc is described below
commit d74fca109dc83580f83c7aaaf427e2da38bce714
Author: libo <[email protected]>
AuthorDate: Tue Jan 20 14:47:33 2026 +0800
Repair the problem that accumulate index used chunkMetadataList (#17040)
* Repair the problem that accumulate index used chunkMetadataList;
Rewrite statistics for mem table.
* Use map instead of list, decrement search complex.
* Optimize codes.
---
.../schemaregion/utils/ResourceByPathUtils.java | 54 ++++++++++++++++------
1 file changed, 40 insertions(+), 14 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
index d7cb2100fa5..aa193aa4e2e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java
@@ -71,6 +71,7 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import static org.apache.iotdb.commons.path.AlignedPath.VECTOR_PLACEHOLDER;
@@ -260,21 +261,17 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
boolean[] exist = new boolean[alignedFullPath.getSchemaList().size()];
boolean modified = false;
boolean isTable = false;
- int index = 0;
+ Map<String, TSDataType> measurementMap =
+ alignedFullPath.getSchemaList().stream()
+ .collect(
+ Collectors.toMap(
+ IMeasurementSchema::getMeasurementName,
IMeasurementSchema::getType));
for (IChunkMetadata chunkMetadata : chunkMetadataList) {
AbstractAlignedChunkMetadata alignedChunkMetadata =
(AbstractAlignedChunkMetadata) chunkMetadata;
isTable = isTable || (alignedChunkMetadata instanceof
TableDeviceChunkMetadata);
modified = (modified || alignedChunkMetadata.isModified());
- TSDataType targetDataType =
alignedFullPath.getSchemaList().get(index).getType();
- if ((alignedChunkMetadata.getValueChunkMetadataList().get(index) != null)
- &&
(alignedChunkMetadata.getValueChunkMetadataList().get(index).getDataType()
- != targetDataType)) {
- // create new statistics object via new data type, and merge
statistics information
- SchemaUtils.rewriteAlignedChunkMetadataStatistics(
- alignedChunkMetadata, index, targetDataType);
- alignedChunkMetadata.setModified(true);
- }
+ rewriteStatistics(alignedChunkMetadata, measurementMap);
if (!useFakeStatistics) {
timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) {
@@ -288,12 +285,10 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
alignedChunkMetadata.getValueChunkMetadataList().get(i).getStatistics());
}
}
- index++;
continue;
}
startTime = Math.min(startTime, chunkMetadata.getStartTime());
endTime = Math.max(endTime, chunkMetadata.getEndTime());
- index++;
}
for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
@@ -309,6 +304,7 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
AbstractAlignedChunkMetadata alignedChunkMetadata =
(AbstractAlignedChunkMetadata) memChunk.getChunkMetaData();
isTable = isTable || (alignedChunkMetadata instanceof
TableDeviceChunkMetadata);
+ rewriteStatistics(alignedChunkMetadata, measurementMap);
if (!useFakeStatistics) {
timeStatistics.mergeStatistics(
alignedChunkMetadata.getTimeChunkMetadata().getStatistics());
@@ -351,6 +347,28 @@ class AlignedResourceByPathUtils extends
ResourceByPathUtils {
: new AlignedTimeSeriesMetadata(timeTimeSeriesMetadata,
valueTimeSeriesMetadataList);
}
+ private static void rewriteStatistics(
+ AbstractAlignedChunkMetadata alignedChunkMetadata, Map<String,
TSDataType> measurementMap) {
+ List<IChunkMetadata> valueChunkMetadataList =
alignedChunkMetadata.getValueChunkMetadataList();
+ for (int i = 0, size = valueChunkMetadataList.size(); i < size; i++) {
+ IChunkMetadata valueChunkMetadata = valueChunkMetadataList.get(i);
+ if (valueChunkMetadata == null) {
+ continue;
+ }
+
+ String measurement = valueChunkMetadata.getMeasurementUid();
+ if (!measurementMap.containsKey(measurement)) {
+ continue;
+ }
+
+ TSDataType targetDataType = measurementMap.get(measurement);
+ if (valueChunkMetadata.getDataType() != targetDataType) {
+
SchemaUtils.rewriteAlignedChunkMetadataStatistics(alignedChunkMetadata, i,
targetDataType);
+ alignedChunkMetadata.setModified(true);
+ }
+ }
+ }
+
@Override
public ReadOnlyMemChunk getReadOnlyMemChunkFromMemTable(
QueryContext context,
@@ -539,10 +557,10 @@ class MeasurementResourceByPathUtils extends
ResourceByPathUtils {
Statistics.getStatsByType(timeSeriesMetadata.getTsDataType());
// flush chunkMetadataList one by one
boolean isModified = false;
+ TSDataType targetDataType = fullPath.getMeasurementSchema().getType();
for (int index = 0; index < chunkMetadataList.size(); index++) {
IChunkMetadata chunkMetadata = chunkMetadataList.get(index);
isModified = (isModified || chunkMetadata.isModified());
- TSDataType targetDataType = fullPath.getMeasurementSchema().getType();
if (chunkMetadata != null && (chunkMetadata.getDataType() !=
targetDataType)) {
// create new statistics object via new data type, and merge
statistics information
SchemaUtils.rewriteNonAlignedChunkMetadataStatistics(
@@ -559,9 +577,17 @@ class MeasurementResourceByPathUtils extends
ResourceByPathUtils {
endTime = Math.max(endTime, chunkMetadata.getEndTime());
}
- for (ReadOnlyMemChunk memChunk : readOnlyMemChunk) {
+ for (int index = 0; index < readOnlyMemChunk.size(); index++) {
+ ReadOnlyMemChunk memChunk = readOnlyMemChunk.get(index);
if (!memChunk.isEmpty()) {
memChunk.sortTvLists();
+ if (memChunk.getChunkMetaData() != null
+ && (memChunk.getChunkMetaData().getDataType() != targetDataType)) {
+ // create new statistics object via new data type, and merge
statistics information
+ SchemaUtils.rewriteNonAlignedChunkMetadataStatistics(
+ chunkMetadataList, index, targetDataType);
+ memChunk.getChunkMetaData().setModified(true);
+ }
if (useFakeStatistics) {
memChunk.initChunkMetaFromTVListsWithFakeStatistics();
startTime = Math.min(startTime,
memChunk.getChunkMetaData().getStartTime());
