Copilot commented on code in PR #19198:
URL: https://github.com/apache/iotdb/pull/19198#discussion_r4559353820
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java:
##########
@@ -375,38 +413,110 @@ private static AbstractAlignedTimeSeriesMetadata
loadAlignedTimeSeriesMetadataFr
} else {
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
- // if all the queried aligned sensors does not exist, we will return
null
- boolean exist = false;
for (String measurement : valueMeasurementList) {
- TimeseriesMetadata valueColumn =
+ valueTimeSeriesMetadataList.add(
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(),
deviceId, measurement),
allSensors,
- context.ignoreNotExistsDevice()
- || resource.getTimeIndexType() ==
ITimeIndex.FILE_TIME_INDEX_TYPE,
+ ignoreNotExists,
isDebug,
context,
- rootMeasurementMetadataIndexNodeOffset);
- exist = (exist || (valueColumn != null));
- valueTimeSeriesMetadataList.add(valueColumn);
- }
- if (!ignoreAllNullRows || exist) {
- alignedTimeSeriesMetadata =
- setModifications(
- resource,
- timeColumn,
- valueTimeSeriesMetadataList,
- alignedPath,
- context,
- globalTimeFilter,
- ignoreAllNullRows);
+ rootMeasurementMetadataIndexNodeOffset));
}
+ alignedTimeSeriesMetadata =
+ buildAlignedTimeSeriesMetadata(
+ resource,
+ alignedPath,
+ timeColumn,
+ valueTimeSeriesMetadataList,
+ context,
+ globalTimeFilter,
+ ignoreAllNullRows);
}
}
return alignedTimeSeriesMetadata;
}
+ static boolean shouldUseDeviceMetadataBatchScan(
+ int selectedColumnCount, int totalFieldColumnCount, boolean debug,
boolean offsetAddressed) {
+ // DEBUG queries keep the per-column cache's diagnostic logs;
offset-addressed external
+ // tsfiles cannot be resolved through the table metadata index; and when
the device's series
+ // fit in a single measurement leaf group, one per-column lookup already
reads them all, so
+ // batching only pays off once the table has more field columns than the
index fan-out
+ return !debug
+ && !offsetAddressed
+ && totalFieldColumnCount
+ >
TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()
+ && (double) selectedColumnCount / totalFieldColumnCount >=
DEVICE_METADATA_BATCH_SCAN_RATIO;
+ }
+
+ private static AbstractAlignedTimeSeriesMetadata
loadAlignedTimeSeriesMetadataBatch(
+ TsFileResource resource,
+ AlignedFullPath alignedPath,
+ List<String> valueMeasurementList,
+ Set<String> allSensors,
+ FragmentInstanceContext context,
+ Filter globalTimeFilter,
+ boolean ignoreAllNullRows,
+ boolean ignoreNotExists,
+ TimeSeriesMetadataCache cache,
+ String filePath,
+ IDeviceID deviceId)
+ throws IOException {
+ Map<String, TimeseriesMetadata> deviceMetadata =
+ cache.getDeviceTimeSeriesMetadata(
+ filePath, resource.getTsFileID(), deviceId, allSensors, context,
ignoreNotExists);
+ if (deviceMetadata.isEmpty()) {
+ // the cache applies the same bloom-filter / ignoreNotExists rules as
the per-column get(),
+ // so an empty map means the device is expected to be absent from this
file
+ return null;
+ }
+ TimeseriesMetadata timeColumn = deviceMetadata.get("");
+ if (timeColumn == null) {
+ return null;
+ }
+ List<TimeseriesMetadata> valueTimeSeriesMetadataList =
+ new ArrayList<>(valueMeasurementList.size());
+ for (String measurement : valueMeasurementList) {
+ valueTimeSeriesMetadataList.add(deviceMetadata.get(measurement));
+ }
+ return buildAlignedTimeSeriesMetadata(
Review Comment:
The new scan test uses a dense tree-model fixture, so it never exercises
`ignoreAllNullRows=false` or a projection whose fields are all absent or empty.
Dropping those all-null rows in the batch path would therefore go undetected.
Add a wide, sparse table-model regression with a dedicated context, compare
timestamps and every field value/null against the per-column scan, and assert
that batching was used.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -293,27 +298,288 @@ public TimeseriesMetadata get(
return new TimeseriesMetadata(timeseriesMetadata);
}
} finally {
-
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
- if (cacheHit) {
- queryContext
- .getQueryStatistics()
- .getLoadTimeSeriesMetadataFromCacheCount()
- .incrementAndGet();
- queryContext
- .getQueryStatistics()
- .getLoadTimeSeriesMetadataFromCacheTime()
- .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime;
+ recordLoadStatistics(
+ queryContext,
+ cacheHit ? 1 : 0,
+ cacheHit ? lookupTime : 0,
+ cacheHit ? 0 : 1,
+ cacheHit ? 0 : lookupTime,
+ loadBloomFilterTime);
+ }
+ }
+
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing
the per-measurement
+ * cache and reading the device in a single batch only for the measurements
that are still
+ * missing. This is used by wide-table scans (a device with many columns) to
avoid walking the
+ * device metadata index once per column / leaf group.
+ *
+ * <p>Measurements already cached are returned without touching the file.
The remaining ones are
+ * first checked against the file's cached bloom filter, so columns absent
from this file do not
+ * force a device read; only measurements that cannot be ruled out are
loaded by the batch read.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * <p>Load statistics are counted per series (not per device read) so that
they stay comparable
+ * with {@link #get}; the bytes read by the batch read cannot be reported
because {@link
+ * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no
IO-size recorder.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long cachePhaseEndTime = 0;
+ long loadBloomFilterTime = 0;
+ long cacheHitCount = 0;
+ long diskLookupCount = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ List<String> missing = new ArrayList<>(allSensors.size());
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ // fast path without locking: reuse everything the per-measurement
cache already has
+ collectCachedMetadata(tsFileID, device, allSensors, result, missing);
+ cacheHitCount = result.size();
+ if (missing.isEmpty()) {
+ cachePhaseEndTime = System.nanoTime();
+ return result;
+ }
+ // serialize reads of the same device+file (same lock key as get) so
threads don't each
+ // deserialize the whole device and race on the shared reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ List<String> stillMissing = new ArrayList<>(missing.size());
+ collectCachedMetadata(tsFileID, device, missing, result,
stillMissing);
+ missing = stillMissing;
+ cacheHitCount = result.size();
+ cachePhaseEndTime = System.nanoTime();
+ if (missing.isEmpty()) {
+ return result;
+ }
+
+ // measurements the cached bloom filter already rejects are known
absent in this file, so
+ // they must not trigger a device read
+ long bloomFilterStartTime = System.nanoTime();
+ BloomFilter bloomFilter =
+ loadBloomFilter(filePath, tsFileID, queryContext,
bloomFilterIoSizeRecorder, false);
+ List<String> possiblyPresent = new ArrayList<>(missing.size());
+ for (String measurement : missing) {
+ if (bloomFilterMayContain(bloomFilter, device, measurement)) {
+ possiblyPresent.add(measurement);
+ } else {
+ result.put(measurement, null);
+ }
+ }
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ diskLookupCount = missing.size();
+ if (possiblyPresent.isEmpty()) {
+ return result;
+ }
+
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ result.putAll(
+ readDeviceTimeSeriesMetadata(
Review Comment:
For a 700-field scan with the time column and 699 fields cached, the one
remaining miss still deserializes the whole device and all its chunk metadata.
This adds reads and allocations compared with the per-column leaf-group lookup.
Re-evaluate batching after cache and bloom-filter resolution, and use a
per-column fallback for small remaining sets.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java:
##########
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.read.reader.series;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.AlignedFullPath;
+import org.apache.iotdb.commons.path.AlignedPath;
+import org.apache.iotdb.commons.path.PartialPath;
+import
org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions;
+import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
+import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache;
+import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
+import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
+import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
+import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.db.utils.constant.TestConstant;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.file.metadata.enums.CompressionType;
+import org.apache.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.tsfile.read.common.TimeRange;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl;
+import org.apache.tsfile.write.chunk.IChunkWriter;
+import org.apache.tsfile.write.schema.IMeasurementSchema;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.apache.tsfile.write.writer.TsFileIOWriter;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+
+import static
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter;
+import static
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk;
+
+/**
+ * Verifies that a real multi-leaf-group device (more field columns than the
measurement index
+ * fan-out) is read correctly through the batch metadata path.
+ */
+public class AlignedSeriesScanBatchTest {
+
+ private static final String DATABASE = "root.wide_pd";
+ private static final IDeviceID DEVICE =
+ IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1");
+
+ @Test
+ public void testBatchScanReadsWideAlignedDevice() throws IOException,
IllegalPathException {
+ int columnCount = 300;
+ File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, 1));
+ if (!file.getParentFile().exists()) {
+ Assert.assertTrue(file.getParentFile().mkdirs());
+ }
+ TsFileResource resource = new TsFileResource(file);
+ List<PartialPath> writtenPaths = new ArrayList<>(columnCount);
+ List<TSDataType> dataTypes = new ArrayList<>(columnCount);
+ List<TSEncoding> encodings = new ArrayList<>(columnCount);
+ List<CompressionType> compressionTypes = new ArrayList<>(columnCount);
+ List<String> measurementNames = new ArrayList<>(columnCount);
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>(columnCount);
+
+ for (int i = 0; i < columnCount; i++) {
+ String measurement = "s" + i;
+ measurementNames.add(measurement);
+ measurementSchemas.add(new MeasurementSchema(measurement,
TSDataType.INT32));
+ writtenPaths.add(new AlignedPath(DEVICE.toString(), measurement));
+ dataTypes.add(TSDataType.INT32);
+ encodings.add(TSEncoding.PLAIN);
+ compressionTypes.add(CompressionType.UNCOMPRESSED);
+ }
+
+ try (TsFileIOWriter writer = new TsFileIOWriter(file)) {
+ writer.startChunkGroup(DEVICE);
+ for (IChunkWriter chunkWriter :
+ createChunkWriter(writtenPaths, dataTypes, encodings,
compressionTypes, true)) {
+ writeAlignedChunk(
+ (AlignedChunkWriterImpl) chunkWriter,
+ writer,
+ Collections.singletonList(new TimeRange(0L, 9L)),
+ true);
+ }
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ resource.updateStartTime(DEVICE, 0);
+ resource.updateEndTime(DEVICE, 9);
+ resource.setStatusForTest(TsFileResourceStatus.NORMAL);
+
+ TimeSeriesMetadataCache metadataCache =
TimeSeriesMetadataCache.getInstance();
+ BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+ metadataCache.clear();
+ bloomFilterCache.clear();
+ try {
+ long cacheHitsBefore =
+ EnvironmentUtils.TEST_QUERY_FI_CONTEXT
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .get();
+ long diskReadsBefore =
+ EnvironmentUtils.TEST_QUERY_FI_CONTEXT
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .get();
+
+ AlignedFullPath scanPath = new AlignedFullPath(DEVICE, measurementNames,
measurementSchemas);
+ SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder();
+ builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList()));
+ SeriesScanOptions scanOptions = builder.build();
+ // all columns are selected and the table is wider than the index
fan-out, so batching applies
+ scanOptions.setTotalFieldColumnCount(columnCount);
+
+ AlignedSeriesScanUtil seriesScanUtil =
+ new AlignedSeriesScanUtil(
+ scanPath, Ordering.ASC, scanOptions,
EnvironmentUtils.TEST_QUERY_FI_CONTEXT);
+ seriesScanUtil.initQueryDataSource(
+ new QueryDataSource(Collections.singletonList(resource),
Collections.emptyList()));
+
+ SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile,
true);
+ SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk,
true);
+ Assert.assertTrue(seriesScanUtil.hasNextPage());
+ TsBlock tsBlock = seriesScanUtil.nextPage();
+ Assert.assertEquals(10, tsBlock.getPositionCount());
Review Comment:
This assertion only verifies the time-column row count. An implementation
that loses the value metadata from the second leaf group can still return ten
rows and satisfy both load-statistics assertions, so the test would miss the
main multi-leaf correctness regression it is intended to catch. Please assert
the timestamps, value-column count, and non-null expected values for every
field, including those in the second leaf group.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -293,27 +298,288 @@ public TimeseriesMetadata get(
return new TimeseriesMetadata(timeseriesMetadata);
}
} finally {
-
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
- if (cacheHit) {
- queryContext
- .getQueryStatistics()
- .getLoadTimeSeriesMetadataFromCacheCount()
- .incrementAndGet();
- queryContext
- .getQueryStatistics()
- .getLoadTimeSeriesMetadataFromCacheTime()
- .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime;
+ recordLoadStatistics(
+ queryContext,
+ cacheHit ? 1 : 0,
+ cacheHit ? lookupTime : 0,
+ cacheHit ? 0 : 1,
+ cacheHit ? 0 : lookupTime,
+ loadBloomFilterTime);
+ }
+ }
+
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing
the per-measurement
+ * cache and reading the device in a single batch only for the measurements
that are still
+ * missing. This is used by wide-table scans (a device with many columns) to
avoid walking the
+ * device metadata index once per column / leaf group.
+ *
+ * <p>Measurements already cached are returned without touching the file.
The remaining ones are
+ * first checked against the file's cached bloom filter, so columns absent
from this file do not
+ * force a device read; only measurements that cannot be ruled out are
loaded by the batch read.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * <p>Load statistics are counted per series (not per device read) so that
they stay comparable
+ * with {@link #get}; the bytes read by the batch read cannot be reported
because {@link
+ * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no
IO-size recorder.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long cachePhaseEndTime = 0;
+ long loadBloomFilterTime = 0;
+ long cacheHitCount = 0;
+ long diskLookupCount = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ List<String> missing = new ArrayList<>(allSensors.size());
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ // fast path without locking: reuse everything the per-measurement
cache already has
+ collectCachedMetadata(tsFileID, device, allSensors, result, missing);
+ cacheHitCount = result.size();
+ if (missing.isEmpty()) {
+ cachePhaseEndTime = System.nanoTime();
+ return result;
+ }
+ // serialize reads of the same device+file (same lock key as get) so
threads don't each
+ // deserialize the whole device and race on the shared reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ List<String> stillMissing = new ArrayList<>(missing.size());
+ collectCachedMetadata(tsFileID, device, missing, result,
stillMissing);
+ missing = stillMissing;
+ cacheHitCount = result.size();
+ cachePhaseEndTime = System.nanoTime();
+ if (missing.isEmpty()) {
+ return result;
+ }
+
+ // measurements the cached bloom filter already rejects are known
absent in this file, so
+ // they must not trigger a device read
+ long bloomFilterStartTime = System.nanoTime();
+ BloomFilter bloomFilter =
+ loadBloomFilter(filePath, tsFileID, queryContext,
bloomFilterIoSizeRecorder, false);
+ List<String> possiblyPresent = new ArrayList<>(missing.size());
+ for (String measurement : missing) {
+ if (bloomFilterMayContain(bloomFilter, device, measurement)) {
+ possiblyPresent.add(measurement);
+ } else {
+ result.put(measurement, null);
+ }
+ }
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ diskLookupCount = missing.size();
+ if (possiblyPresent.isEmpty()) {
+ return result;
+ }
Review Comment:
When the time column resolves to null from the cache or bloom filter, a
selected field's bloom-filter false positive can still make `possiblyPresent`
nonempty. The batch path then unnecessarily reads an absent device and may
throw `IOException`, while the existing aligned path skips the file. Return
before the device read when the requested time column is null, and add a
mixed-result bloom-filter regression.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -293,27 +298,288 @@ public TimeseriesMetadata get(
return new TimeseriesMetadata(timeseriesMetadata);
}
} finally {
-
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
- if (cacheHit) {
- queryContext
- .getQueryStatistics()
- .getLoadTimeSeriesMetadataFromCacheCount()
- .incrementAndGet();
- queryContext
- .getQueryStatistics()
- .getLoadTimeSeriesMetadataFromCacheTime()
- .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ long lookupTime = System.nanoTime() - startTime - loadBloomFilterTime;
+ recordLoadStatistics(
+ queryContext,
+ cacheHit ? 1 : 0,
+ cacheHit ? lookupTime : 0,
+ cacheHit ? 0 : 1,
+ cacheHit ? 0 : lookupTime,
+ loadBloomFilterTime);
+ }
+ }
+
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reusing
the per-measurement
+ * cache and reading the device in a single batch only for the measurements
that are still
+ * missing. This is used by wide-table scans (a device with many columns) to
avoid walking the
+ * device metadata index once per column / leaf group.
+ *
+ * <p>Measurements already cached are returned without touching the file.
The remaining ones are
+ * first checked against the file's cached bloom filter, so columns absent
from this file do not
+ * force a device read; only measurements that cannot be ruled out are
loaded by the batch read.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * <p>Load statistics are counted per series (not per device read) so that
they stay comparable
+ * with {@link #get}; the bytes read by the batch read cannot be reported
because {@link
+ * TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)} has no
IO-size recorder.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long cachePhaseEndTime = 0;
+ long loadBloomFilterTime = 0;
+ long cacheHitCount = 0;
+ long diskLookupCount = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ List<String> missing = new ArrayList<>(allSensors.size());
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ // fast path without locking: reuse everything the per-measurement
cache already has
+ collectCachedMetadata(tsFileID, device, allSensors, result, missing);
+ cacheHitCount = result.size();
+ if (missing.isEmpty()) {
+ cachePhaseEndTime = System.nanoTime();
+ return result;
+ }
+ // serialize reads of the same device+file (same lock key as get) so
threads don't each
+ // deserialize the whole device and race on the shared reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ List<String> stillMissing = new ArrayList<>(missing.size());
+ collectCachedMetadata(tsFileID, device, missing, result,
stillMissing);
+ missing = stillMissing;
+ cacheHitCount = result.size();
+ cachePhaseEndTime = System.nanoTime();
+ if (missing.isEmpty()) {
+ return result;
+ }
+
+ // measurements the cached bloom filter already rejects are known
absent in this file, so
+ // they must not trigger a device read
+ long bloomFilterStartTime = System.nanoTime();
+ BloomFilter bloomFilter =
+ loadBloomFilter(filePath, tsFileID, queryContext,
bloomFilterIoSizeRecorder, false);
+ List<String> possiblyPresent = new ArrayList<>(missing.size());
+ for (String measurement : missing) {
+ if (bloomFilterMayContain(bloomFilter, device, measurement)) {
+ possiblyPresent.add(measurement);
+ } else {
+ result.put(measurement, null);
+ }
+ }
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ diskLookupCount = missing.size();
+ if (possiblyPresent.isEmpty()) {
+ return result;
+ }
+
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ result.putAll(
+ readDeviceTimeSeriesMetadata(
+ reader,
+ filePath,
+ tsFileID,
+ device,
+ new HashSet<>(possiblyPresent),
+ true,
+ ignoreNotExists));
+ return result;
+ }
+ }
+
+ // cache disabled / external tsfiles: probe the time column first so an
absent device does
+ // not pay for the whole-device read
+ long bloomFilterStartTime = System.nanoTime();
+ BloomFilter bloomFilter =
+ loadBloomFilter(
+ filePath, tsFileID, queryContext, bloomFilterIoSizeRecorder,
externalTsFile);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ diskLookupCount = allSensors.size();
+ if (!bloomFilterMayContain(bloomFilter, device, "")) {
+ for (String measurement : allSensors) {
+ result.put(measurement, null);
+ }
+ return result;
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, filePath, tsFileID, device, allSensors, false,
ignoreNotExists);
+ } finally {
+ long endTime = System.nanoTime();
+ long cacheLookupTime = cachePhaseEndTime == 0 ? 0 : cachePhaseEndTime -
startTime;
+ long diskLookupTime =
+ cachePhaseEndTime == 0
+ ? endTime - startTime - loadBloomFilterTime
+ : endTime - cachePhaseEndTime - loadBloomFilterTime;
Review Comment:
On a cold batch, `cacheHitCount` is zero, so `recordLoadStatistics()` drops
the cache phase, while this calculation also excludes it from disk time. Cache
probes and the device-lock wait therefore disappear from metadata-load timings.
Include that phase in disk lookup time when there are no cache hits.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]