This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch ignoreValuePageInNonAlignedSeriesScanProcess in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e18a93641483f3f6d1e904a2a3e041fdff8ec0a4 Author: shuwenwei <[email protected]> AuthorDate: Thu Jan 8 11:29:28 2026 +0800 Detect inconsistent chunk type when querying or compacting tsfile --- .../exception/ChunkTypeInconsistentException.java | 42 ++++++++++++++++++++++ .../execution/fragment/QueryStatistics.java | 10 +++++- .../execution/operator/source/FileLoaderUtils.java | 9 ++++- .../FragmentInstanceStatisticsDrawer.java | 6 ++++ .../FastNonAlignedSeriesCompactionExecutor.java | 11 ++++++ .../readchunk/SingleSeriesCompactionExecutor.java | 8 +++++ .../read/reader/chunk/DiskChunkLoader.java | 21 +++++++++++ .../src/main/thrift/datanode.thrift | 2 ++ 8 files changed, 107 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/ChunkTypeInconsistentException.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/ChunkTypeInconsistentException.java new file mode 100644 index 00000000000..cd2bfc025f6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/exception/ChunkTypeInconsistentException.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.exception; + +import org.apache.tsfile.file.metadata.IDeviceID; + +import java.io.IOException; + +public class ChunkTypeInconsistentException extends IOException { + + public ChunkTypeInconsistentException() {} + + public ChunkTypeInconsistentException( + String filePath, IDeviceID deviceID, String measurement, long offsetOfChunk) { + super( + "Unexpected chunk type detected when reading non-aligned chunk reader. File path: " + + filePath + + ", offsetOfChunk: " + + offsetOfChunk + + ", deviceID: " + + deviceID + + ", measurement: " + + measurement); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java index 277e9106520..5291f96dfb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java @@ -92,6 +92,9 @@ public class QueryStatistics { // statistics for count and time of page decode private final AtomicLong pageReaderMaxUsedMemorySize = new AtomicLong(0); + // statistics for count of chunk with metadata errors + private final AtomicLong chunkWithMetadataErrorsCount = new AtomicLong(0); + public AtomicLong getLoadTimeSeriesMetadataDiskSeqCount() { return loadTimeSeriesMetadataDiskSeqCount; } @@ -288,6 +291,10 @@ public class QueryStatistics { return loadTimeSeriesMetadataFromDiskTime; } + public AtomicLong getChunkWithMetadataErrorsCount() { + return chunkWithMetadataErrorsCount; + } + public TQueryStatistics toThrift() { return new TQueryStatistics( loadTimeSeriesMetadataDiskSeqCount.get(), @@ -336,6 +343,7 @@ public class QueryStatistics { loadTimeSeriesMetadataActualIOSize.get(), loadChunkFromCacheCount.get(), loadChunkFromDiskCount.get(), - loadChunkActualIOSize.get()); + loadChunkActualIOSize.get(), + chunkWithMetadataErrorsCount.get()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index a92796dacfb..d34abd60946 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.path.AlignedFullPath; import org.apache.iotdb.commons.path.NonAlignedFullPath; +import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; @@ -488,7 +489,13 @@ public class FileLoaderUtils { chunkMetaData.setModified(true); } IChunkLoader chunkLoader = chunkMetaData.getChunkLoader(); - chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter); + try { + chunkReader = chunkLoader.getChunkReader(chunkMetaData, globalTimeFilter); + } catch (ChunkTypeInconsistentException e) { + // if the chunk in tsfile is a value chunk of aligned series, we should skip all data of + // this chunk. + return Collections.emptyList(); + } } return isModified diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java index 7a8b7097eb9..5961225cdb9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/statistics/FragmentInstanceStatisticsDrawer.java @@ -431,6 +431,12 @@ public class FragmentInstanceStatisticsDrawer { 2, "pageReaderMaxUsedMemorySize", queryStatistics.pageReaderMaxUsedMemorySize); + + addLineWithValueCheck( + singleFragmentInstanceArea, + 2, + "chunkWithMetadataErrorsCount", + queryStatistics.chunkWithMetadataErrorsCount); } private void addLine(List<StatisticLine> resultForSingleInstance, int level, String value) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java index 697376c233a..363554fb606 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.ex import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PatternTreeMap; +import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.exception.WriteProcessException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus; @@ -38,6 +39,7 @@ import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory; import org.apache.tsfile.encrypt.EncryptUtils; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.PageException; +import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.header.PageHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; @@ -221,6 +223,15 @@ public class FastNonAlignedSeriesCompactionExecutor extends SeriesCompactionExec .get(chunkMetadataElement.fileElement.resource) .readMemChunk((ChunkMetadata) chunkMetadataElement.chunkMetadata); } + byte chunkType = chunkMetadataElement.chunk.getHeader().getChunkType(); + if (chunkType != MetaMarker.CHUNK_HEADER + && chunkType != MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { + throw new ChunkTypeInconsistentException( + chunkMetadataElement.fileElement.resource.getTsFilePath(), + deviceId, + chunkMetadataElement.chunkMetadata.getMeasurementUid(), + chunkMetadataElement.chunkMetadata.getOffsetOfChunkHeader()); + } if (!hasStartMeasurement) { // for nonAligned sensors, only after getting chunkMetadatas can we create metadata to // start diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java index f4e9e1bc21e..9ffd2b7044d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/readchunk/SingleSeriesCompactionExecutor.java @@ -20,12 +20,14 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.readchunk; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary; import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileWriter; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.tsfile.encrypt.EncryptUtils; +import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.header.ChunkHeader; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -134,6 +136,12 @@ public class SingleSeriesCompactionExecutor { chunkMetadata.getNewType() != null ? reader.readMemChunk(chunkMetadata).rewrite(chunkMetadata.getNewType()) : reader.readMemChunk(chunkMetadata); + byte chunkType = currentChunk.getHeader().getChunkType(); + if (chunkType != MetaMarker.CHUNK_HEADER + && chunkType != MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { + throw new ChunkTypeInconsistentException( + reader.getFileName(), device, measurement, chunkMetadata.getOffsetOfChunkHeader()); + } summary.increaseProcessChunkNum(1); summary.increaseProcessPointNum(chunkMetadata.getNumOfPoints()); if (chunkMetadata.getNewType() != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java index a2ff24233d9..84b976bc299 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/chunk/DiskChunkLoader.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk; +import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.storageengine.buffer.ChunkCache; @@ -27,6 +28,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.utils.ObjectTypeUtils; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.MetaMarker; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.read.common.Chunk; @@ -34,6 +36,8 @@ import org.apache.tsfile.read.controller.IChunkLoader; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.read.reader.IChunkReader; import org.apache.tsfile.read.reader.chunk.ChunkReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; @@ -42,6 +46,8 @@ import static org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.INI /** To read one chunk from disk, and only used in iotdb server module. */ public class DiskChunkLoader implements IChunkLoader { + private static final Logger LOGGER = LoggerFactory.getLogger(DiskChunkLoader.class); + private final QueryContext context; private final TsFileResource resource; @@ -86,6 +92,21 @@ public class DiskChunkLoader implements IChunkLoader { chunkMetaData.getDeleteIntervalList(), chunkMetaData.getStatistics(), context); + byte chunkType = chunk.getHeader().getChunkType(); + if (chunkType != MetaMarker.CHUNK_HEADER + && chunkType != MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) { + if (context.getQueryStatistics().getChunkWithMetadataErrorsCount().getAndAdd(1) == 0) { + LOGGER.warn( + "Unexpected chunk type detected when reading non-aligned chunk reader. " + + "The chunk metadata indicates a non-aligned chunk, " + + "but the actual chunk read from tsfile is a value chunk of aligned series. " + + "tsFile={}, chunkOffset={}, chunkType={}", + resource.getTsFilePath(), + chunkMetaData.getOffsetOfChunkHeader(), + chunkType); + } + throw new ChunkTypeInconsistentException(); + } final TsFileID tsFileID = getTsFileID(); if (tsFileID.regionId > 0 && chunkMetaData.getDataType() == TSDataType.OBJECT) { diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 942a04f5f2a..585e5e36be5 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -749,6 +749,8 @@ struct TQueryStatistics { 45: i64 loadChunkFromCacheCount 46: i64 loadChunkFromDiskCount 47: i64 loadChunkActualIOSize + + 48: i64 chunkWithMetadataErrorsCount }
