This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch fixSetModified-0205 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 53d7c03b84c3ad70ecf15a1433fcf4c50d0f95e4 Author: shuwenwei <[email protected]> AuthorDate: Thu Feb 5 18:26:57 2026 +0800 fix setModified --- .../execution/operator/source/FileLoaderUtils.java | 68 ++++------- .../execution/operator/source/SeriesScanUtil.java | 33 +----- .../org/apache/iotdb/db/utils/SchemaUtils.java | 132 +++++++-------------- pom.xml | 2 +- 4 files changed, 74 insertions(+), 161 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 9b341dc9971..3087108d1f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -113,13 +113,11 @@ public class FileLoaderUtils { context.isDebug(), context); if (timeSeriesMetadata != null) { - SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType()); long t2 = System.nanoTime(); List<ModEntry> pathModifications = context.getPathModifications( resource, seriesPath.getDeviceId(), seriesPath.getMeasurement()); - timeSeriesMetadata.setModified( - timeSeriesMetadata.isModified() || !pathModifications.isEmpty()); + timeSeriesMetadata.setModified(!pathModifications.isEmpty()); timeSeriesMetadata.setChunkMetadataLoader( new DiskChunkMetadataLoader(resource, context, globalTimeFilter, pathModifications)); int modificationCount = pathModifications.size(); @@ -141,13 +139,14 @@ public class FileLoaderUtils { timeSeriesMetadata = (TimeseriesMetadata) resource.getTimeSeriesMetadata(seriesPath, globalTimeFilter); if (timeSeriesMetadata != null) { - SchemaUtils.changeMetadataModified(timeSeriesMetadata, seriesPath.getSeriesType()); timeSeriesMetadata.setChunkMetadataLoader( new MemChunkMetadataLoader(resource, seriesPath, context, globalTimeFilter)); } } if (timeSeriesMetadata != null) { + SchemaUtils.changeTimeseriesMetadataModified( + timeSeriesMetadata, seriesPath.getSeriesType()); if (timeSeriesMetadata.getStatistics().getStartTime() > timeSeriesMetadata.getStatistics().getEndTime()) { return null; @@ -209,15 +208,12 @@ public class FileLoaderUtils { alignedTimeSeriesMetadata = loadAlignedTimeSeriesMetadataFromDisk( resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows); - SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList); } else { // if the tsfile is unclosed, we just get it directly from TsFileResource loadFromMem = true; alignedTimeSeriesMetadata = (AbstractAlignedTimeSeriesMetadata) resource.getTimeSeriesMetadata(alignedPath, globalTimeFilter); if (alignedTimeSeriesMetadata != null) { - SchemaUtils.changeAlignedMetadataModified(alignedTimeSeriesMetadata, targetDataTypeList); - alignedTimeSeriesMetadata.setChunkMetadataLoader( new MemAlignedChunkMetadataLoader( resource, alignedPath, context, globalTimeFilter, ignoreAllNullRows)); @@ -226,6 +222,8 @@ public class FileLoaderUtils { } if (alignedTimeSeriesMetadata != null) { + SchemaUtils.changeAlignedTimeseriesMetadataModified( + alignedTimeSeriesMetadata, targetDataTypeList); if (alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getStartTime() > alignedTimeSeriesMetadata.getTimeseriesMetadata().getStatistics().getEndTime()) { return null; @@ -320,25 +318,18 @@ public class FileLoaderUtils { new ArrayList<>(valueMeasurementList.size()); // if all the queried aligned sensors does not exist, we will return null boolean exist = false; - int i = 0; - for (String valueMeasurement : valueMeasurementList) { + for (String measurement : valueMeasurementList) { TimeseriesMetadata valueColumn = cache.get( filePath, - new TimeSeriesMetadataCacheKey( - resource.getTsFileID(), deviceId, valueMeasurement), + new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), allSensors, context.ignoreNotExistsDevice() || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, isDebug, context); exist = (exist || (valueColumn != null)); - if (valueColumn != null) { - SchemaUtils.changeAlignedMetadataModified( - valueColumn, alignedPath.getSchemaList().get(i).getType()); - } valueTimeSeriesMetadataList.add(valueColumn); - i++; } if (!ignoreAllNullRows || exist) { alignedTimeSeriesMetadata = @@ -458,36 +449,33 @@ public class FileLoaderUtils { * IOException will be thrown */ public static List<IPageReader> loadPageReaderList( - IChunkMetadata chunkMetaData, - Filter globalTimeFilter, - boolean isAligned, - List<TSDataType> targetDataTypeList) + IChunkMetadata chunkMetaData, Filter globalTimeFilter, List<TSDataType> targetDataTypeList) throws IOException { checkArgument(chunkMetaData != null, "Can't init null chunkMeta"); IChunkReader chunkReader; - boolean isModified = false; - if (isAligned) { + boolean cannotUseStatisticsAfterAlter = false; + if (chunkMetaData instanceof AbstractAlignedChunkMetadata) { AbstractAlignedChunkMetadata alignedChunkMetadata = (AbstractAlignedChunkMetadata) chunkMetaData; - for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { - if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { - if (!SchemaUtils.isUsingSameStatistics( - alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(), - targetDataTypeList.get(i)) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) { - isModified = true; - alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true); + if (alignedChunkMetadata.isModified()) { + for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { + if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { + if (!SchemaUtils.canUseStatisticsAfterAlter( + alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(), + targetDataTypeList.get(i))) { + cannotUseStatisticsAfterAlter = true; + } } } } IChunkLoader chunkLoader = alignedChunkMetadata.getChunkLoader(); chunkReader = chunkLoader.getChunkReader(alignedChunkMetadata, globalTimeFilter); } else { - if (!SchemaUtils.isUsingSameStatistics(chunkMetaData.getDataType(), targetDataTypeList.get(0)) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(0))) { - isModified = true; - chunkMetaData.setModified(true); + if (chunkMetaData.isModified() + && !SchemaUtils.canUseStatisticsAfterAlter( + chunkMetaData.getDataType(), targetDataTypeList.get(0))) { + cannotUseStatisticsAfterAlter = true; } IChunkLoader chunkLoader = chunkMetaData.getChunkLoader(); try { @@ -499,14 +487,10 @@ public class FileLoaderUtils { } } - return isModified - ? chunkReader.loadPageReaderList().stream() - .peek( - iPageReader -> { - iPageReader.setModified(true); - }) - .collect(Collectors.toList()) - : chunkReader.loadPageReaderList(); + if (cannotUseStatisticsAfterAlter) { + chunkReader.loadPageReaderList().forEach(iPageReader -> iPageReader.setModified(true)); + } + return chunkReader.loadPageReaderList(); } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java index 1e1cffba5ff..cba5f1fa6c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java @@ -453,15 +453,6 @@ public class SeriesScanUtil implements Accountable { orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics())); unpackAllOverlappedTimeSeriesMetadataToCachedChunkMetadata( orderUtils.getOverlapCheckTime(firstChunkMetadata.getStatistics()), false); - if (isAligned) { - SchemaUtils.changeAlignedMetadataModified( - (AbstractAlignedChunkMetadata) firstChunkMetadata, - firstChunkMetadata.getDataType(), - getTsDataTypeList()); - } else { - SchemaUtils.changeMetadataModified( - firstChunkMetadata, firstChunkMetadata.getDataType(), dataType); - } if (firstChunkMetadata.equals(cachedChunkMetadata.peek())) { firstChunkMetadata = cachedChunkMetadata.poll(); break; @@ -489,15 +480,6 @@ public class SeriesScanUtil implements Accountable { if (init && firstChunkMetadata == null && !cachedChunkMetadata.isEmpty()) { firstChunkMetadata = cachedChunkMetadata.poll(); - if (isAligned) { - SchemaUtils.changeAlignedMetadataModified( - (AbstractAlignedChunkMetadata) firstChunkMetadata, - firstChunkMetadata.getDataType(), - getTsDataTypeList()); - } else { - SchemaUtils.changeMetadataModified( - firstChunkMetadata, firstChunkMetadata.getDataType(), dataType); - } } } @@ -510,19 +492,16 @@ public class SeriesScanUtil implements Accountable { AbstractAlignedChunkMetadata alignedChunkMetadata = (AbstractAlignedChunkMetadata) chunkMetadata; for (int i = 0; i < alignedChunkMetadata.getValueChunkMetadataList().size(); i++) { - if ((alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) - && !SchemaUtils.isUsingSameStatistics( + if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null + && !SchemaUtils.canUseStatisticsAfterAlter( alignedChunkMetadata.getValueChunkMetadataList().get(i).getDataType(), - getTsDataTypeList().get(i)) - && !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(i))) { + getTsDataTypeList().get(i))) { alignedChunkMetadata.getValueChunkMetadataList().get(i).setModified(true); } } - chunkMetadata = alignedChunkMetadata; } else if (chunkMetadata instanceof ChunkMetadata) { - if (!SchemaUtils.isUsingSameStatistics( - chunkMetadata.getDataType(), getTsDataTypeList().get(0)) - && !SchemaUtils.canUseStatisticsAfterAlter(getTsDataTypeList().get(0))) { + if (!SchemaUtils.canUseStatisticsAfterAlter( + chunkMetadata.getDataType(), getTsDataTypeList().get(0))) { chunkMetadata.setModified(true); } } @@ -704,7 +683,7 @@ public class SeriesScanUtil implements Accountable { } List<IPageReader> pageReaderList = FileLoaderUtils.loadPageReaderList( - chunkMetaData, scanOptions.getGlobalTimeFilter(), isAligned, getTsDataTypeList()); + chunkMetaData, scanOptions.getGlobalTimeFilter(), getTsDataTypeList()); // init TsBlockBuilder for each page reader pageReaderList.forEach(p -> p.initTsBlockBuilder(getTsDataTypeList())); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 6044b2331d1..b44952472f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -40,6 +40,9 @@ import org.apache.tsfile.utils.Binary; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; +import javax.validation.constraints.NotNull; + import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collections; @@ -314,120 +317,67 @@ public class SchemaUtils { dataTypeColumnClassMap.get(originalDataType), dataTypeColumnClassMap.get(dataType)); } - public static boolean isUsingSameStatistics(TSDataType originalDataType, TSDataType dataType) { - if (originalDataType == dataType) { - return true; - } - if (!dataTypeColumnStatisticsClassMap.containsKey(originalDataType) - || !dataTypeColumnStatisticsClassMap.containsKey(dataType)) { - return false; - } - return Objects.equals( - dataTypeColumnStatisticsClassMap.get(originalDataType), - dataTypeColumnStatisticsClassMap.get(dataType)); - } - - public static boolean canUseStatisticsAfterAlter(TSDataType dataType) { - return !canNotUseStatisticAfterAlterClassSet.contains(dataType); - } - - public static void changeMetadataModified( - TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) { + public static void changeTimeseriesMetadataModified( + @Nullable TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) { if (timeseriesMetadata == null) { return; } - if (!SchemaUtils.isUsingSameStatistics(timeseriesMetadata.getTsDataType(), targetDataType) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataType)) { - timeseriesMetadata.setModified(true); - List<IChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata != null) { - chunkMetadata.setModified(true); - } - } - } + if (!SchemaUtils.canUseStatisticsAfterAlter( + timeseriesMetadata.getTsDataType(), targetDataType)) { + markAllChunkMetadataModified(timeseriesMetadata); } } - public static void changeAlignedMetadataModified( - AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata, + public static void changeAlignedTimeseriesMetadataModified( + @Nullable AbstractAlignedTimeSeriesMetadata alignedTimeSeriesMetadata, List<TSDataType> targetDataTypeList) { if (alignedTimeSeriesMetadata == null) { return; } - - int i = 0; - for (TimeseriesMetadata timeseriesMetadata : - alignedTimeSeriesMetadata.getValueTimeseriesMetadataList()) { - if ((timeseriesMetadata != null) - && !SchemaUtils.isUsingSameStatistics( - timeseriesMetadata.getTsDataType(), targetDataTypeList.get(i)) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) { - timeseriesMetadata.setModified(true); - alignedTimeSeriesMetadata.setModified(true); - List<IChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata != null) { - chunkMetadata.setModified(true); - } - } - } + for (int i = 0; i < alignedTimeSeriesMetadata.getValueTimeseriesMetadataList().size(); i++) { + TimeseriesMetadata valueTimeseriesMetadata = + alignedTimeSeriesMetadata.getValueTimeseriesMetadataList().get(i); + if (valueTimeseriesMetadata != null + && !SchemaUtils.canUseStatisticsAfterAlter( + valueTimeseriesMetadata.getTsDataType(), targetDataTypeList.get(i))) { + markAllChunkMetadataModified(alignedTimeSeriesMetadata.getTimeseriesMetadata()); + return; } - i++; } } - public static void changeAlignedMetadataModified( - TimeseriesMetadata timeseriesMetadata, TSDataType targetDataType) { - if (timeseriesMetadata == null) { - return; - } - - if (!SchemaUtils.isUsingSameStatistics(timeseriesMetadata.getTsDataType(), targetDataType) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataType)) { - timeseriesMetadata.setModified(true); - List<IChunkMetadata> chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - for (IChunkMetadata chunkMetadata : chunkMetadataList) { - if (chunkMetadata != null) { - chunkMetadata.setModified(true); - } - } + private static void markAllChunkMetadataModified(@NotNull TimeseriesMetadata timeseriesMetadata) { + timeseriesMetadata.setModified(true); + for (IChunkMetadata iChunkMetadata : timeseriesMetadata.getChunkMetadataList()) { + if (iChunkMetadata != null) { + iChunkMetadata.setModified(true); } } } - public static void changeMetadataModified( - IChunkMetadata chunkMetadata, TSDataType sourceDataType, TSDataType targetDataType) { - if (chunkMetadata == null) { - return; - } - if (!SchemaUtils.isUsingSameStatistics(sourceDataType, targetDataType) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataType)) { - chunkMetadata.setModified(true); + public static boolean canUseStatisticsAfterAlter( + TSDataType originalDataType, TSDataType targetDataType) { + if (isUsingSameStatistics(originalDataType, targetDataType)) { + return true; } + return canUseStatisticsAfterAlter(targetDataType); } - public static void changeAlignedMetadataModified( - AbstractAlignedChunkMetadata chunkMetadata, - TSDataType sourceDataType, - List<TSDataType> targetDataTypeList) { - if (chunkMetadata == null) { - return; + private static boolean isUsingSameStatistics(TSDataType originalDataType, TSDataType dataType) { + if (originalDataType == dataType) { + return true; } - int i = 0; - for (IChunkMetadata iChunkMetadata : chunkMetadata.getValueChunkMetadataList()) { - if ((iChunkMetadata != null) - && !SchemaUtils.isUsingSameStatistics( - iChunkMetadata.getDataType(), targetDataTypeList.get(i)) - && !SchemaUtils.canUseStatisticsAfterAlter(targetDataTypeList.get(i))) { - iChunkMetadata.setModified(true); - chunkMetadata.setModified(true); - } - i++; + if (!dataTypeColumnStatisticsClassMap.containsKey(originalDataType) + || !dataTypeColumnStatisticsClassMap.containsKey(dataType)) { + return false; } + return Objects.equals( + dataTypeColumnStatisticsClassMap.get(originalDataType), + dataTypeColumnStatisticsClassMap.get(dataType)); + } + + private static boolean canUseStatisticsAfterAlter(TSDataType dataType) { + return !canNotUseStatisticAfterAlterClassSet.contains(dataType); } public static void rewriteAlignedChunkMetadataStatistics( diff --git a/pom.xml b/pom.xml index 9be2fa2359a..bf41ba90d43 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ <thrift.version>0.14.1</thrift.version> <xz.version>1.9</xz.version> <zstd-jni.version>1.5.6-3</zstd-jni.version> - <tsfile.version>2.2.1-260203-SNAPSHOT</tsfile.version> + <tsfile.version>2.2.1-260205-SNAPSHOT</tsfile.version> </properties> <!-- if we claim dependencies in dependencyManagement, then we do not claim
