JackieTien97 commented on code in PR #19198:
URL: https://github.com/apache/iotdb/pull/19198#discussion_r4558365667
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ QueryContext queryContext,
+ LongConsumer bloomFilterIoSizeRecorder)
+ throws IOException {
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+ false,
+ bloomFilterIoSizeRecorder,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+ false);
+ return bloomFilter == null
+ || bloomFilter.contains(device.toString() +
IoTDBConstant.PATH_SEPARATOR);
+ }
+
+ /**
+ * @return copies of the cached metadata for every measurement in {@code
allSensors}, or null as
+ * soon as one of them is not cached and the device therefore has to be
read from disk
+ */
+ private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+ TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ for (String measurement : allSensors) {
+ TimeseriesMetadata metadata =
+ lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID,
device, measurement));
+ if (metadata == null) {
+ return null;
+ }
+ result.put(
+ measurement,
+ metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new
TimeseriesMetadata(metadata));
+ }
+ return result;
+ }
+
+ private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+ TsFileSequenceReader reader,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ boolean cacheEnabled,
+ boolean ignoreNotExists)
+ throws IOException {
+ // Use the chunk-metadata-including variant: the returned
TimeseriesMetadata are passed to
+ // setModifications and then to DiskAlignedChunkMetadataLoader, which
dereferences the chunk
+ // metadata list, so readDeviceMetadata (which skips chunk metadata) would
NPE at query time.
+ List<TimeseriesMetadata> deviceMetadataList =
reader.getDeviceTimeseriesMetadata(device);
+ if (deviceMetadataList.isEmpty()) {
+ // the caller already consulted the bloom filter, so the device could
not be ruled out:
+ // missing metadata is an inconsistency unless ignoreNotExists is set
+ if (ignoreNotExists) {
+ return Collections.emptyMap();
+ }
+ throw new IOException(
+
String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT,
device));
Review Comment:
**[P1] Wrong diagnostic for a device missing from the TsFile metadata**
This path is reached when the resource's device-level time index says the
device is in the file but the TsFile metadata does not contain it.
`DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT` reads `%s does not exist in the
resource file`. That constant is used by `RepairDataFileScanUtil` for the
opposite case: a device that is in the TsFile but not in the resource. It also
drops the file path. In a probe the cause message was `root.absent does not
exist in the resource file`. The per-column path reports TsFile's `Device
{root.absent} is not in tsFileMetaData of <file>`.
Someone investigating the resulting `CorruptedTsFileException` would be sent
to the `.resource` file. Could you add a dedicated constant to both the `en`
and `zh` `StorageEngineMessages`, with the device and the TsFile path, e.g.
`Device %s is recorded in the resource but missing from the metadata of TsFile
%s`?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
Review Comment:
**[P1] All-or-nothing cache check turns non-cacheable absent columns into a
full device re-read on every query**
`getCachedDeviceTimeSeriesMetadata` returns null as soon as one selected
sensor is missing from the LRU. When a selected column is absent from the
device in this file, `readDeviceTimeSeriesMetadata` only caches the
`NULL_EXISTS_CACHE_PLACE_HOLDER` if `!ignoreNotExists &&
isMayCacheNonExistSeries()` is true. Otherwise the cache check fails on every
query, and every query re-reads and re-deserializes the whole device, including
the chunk metadata of every column. `ignoreNotExists` is true for files whose
time index was degraded to `FILE_TIME_INDEX_TYPE` and for queries during
startup recovery (`ignoreNotExistsDevice()`).
The per-column `get()` handles the same case cheaply. Present columns stay
cached, and absent columns are rejected by the cached per-measurement
BloomFilter (`device.measurement`) without touching the file, barring false
positives.
I reproduced this with a mocked reader. The selected set is `{"", s1, s2}`,
`s2` is absent from the file, and I ran 5 queries:
- `ignoreNotExists=true`: batch `getDeviceTimeseriesMetadata` x5 vs
per-column `readTimeseriesMetadata` x1
- `ignoreNotExists=false`, `may_cache_nonexist_series=false`: x5 vs x1
- defaults: x1 vs x1
Sparse wide tables and columns added after older files were written make
this common for `SELECT *`. A single evicted entry has a similar effect: it
forces all leaf groups of the device to be re-read, while the per-column path
re-reads only one.
Possible fixes: when the cache check fails, probe the BloomFilter for the
missing keys and treat filtered-out ones as known-absent. Alternatively, fetch
only the missing measurements through the per-column `get()` and use the batch
read only when many keys are missing.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ QueryContext queryContext,
+ LongConsumer bloomFilterIoSizeRecorder)
+ throws IOException {
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+ false,
+ bloomFilterIoSizeRecorder,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+ false);
+ return bloomFilter == null
+ || bloomFilter.contains(device.toString() +
IoTDBConstant.PATH_SEPARATOR);
+ }
+
+ /**
+ * @return copies of the cached metadata for every measurement in {@code
allSensors}, or null as
+ * soon as one of them is not cached and the device therefore has to be
read from disk
+ */
+ private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+ TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ for (String measurement : allSensors) {
+ TimeseriesMetadata metadata =
+ lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID,
device, measurement));
+ if (metadata == null) {
+ return null;
+ }
+ result.put(
+ measurement,
+ metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new
TimeseriesMetadata(metadata));
+ }
+ return result;
+ }
+
+ private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+ TsFileSequenceReader reader,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ boolean cacheEnabled,
+ boolean ignoreNotExists)
+ throws IOException {
+ // Use the chunk-metadata-including variant: the returned
TimeseriesMetadata are passed to
+ // setModifications and then to DiskAlignedChunkMetadataLoader, which
dereferences the chunk
+ // metadata list, so readDeviceMetadata (which skips chunk metadata) would
NPE at query time.
+ List<TimeseriesMetadata> deviceMetadataList =
reader.getDeviceTimeseriesMetadata(device);
Review Comment:
**[P2] Metadata IO is not recorded, and load counts are now per device**
`TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)` has no IO-size
recorder. Its internal `readData` calls don't record anything, and the
`timeSeriesMetadataIoSizeRecorder` passed to `FileReaderManager.get` only
covers opening the reader. So bytes read by the batch path never reach
`loadTimeSeriesMetadataActualIOSize`.
In addition, `LoadTimeSeriesMetadataFromCacheCount/FromDiskCount` now go up
by 1 per device and file instead of 1 per series. For an 800-column device that
is roughly 801 lookups versus 1. `AlignedSeriesScanBatchTest` asserts this new
counting (`diskReadsBefore + 2`). As a result, `EXPLAIN ANALYZE` numbers for
the same query change meaning depending on which side of the 0.6 threshold it
falls.
Could the batch read record its size? For example, sum the buffer sizes, or
add a recorder-taking overload in TsFile. Please also either keep per-series
counting or document the change.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
Review Comment:
**[P3] Cache hit ratio is inflated on a miss**
On a miss, `getCachedDeviceTimeSeriesMetadata` runs twice, once outside and
once inside the lock. Each run calls `lruCache.getIfPresent` (the cache uses
`recordStats()`) for every key it checks before the first missing one. Those
hits are recorded twice, so `calculateTimeSeriesMetadataHitRatio()` reads high
for wide scans. Checking presence once, or only on the second run, and copying
entries only after every key has hit would avoid this.
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java:
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.read.reader.series;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.AlignedFullPath;
+import
org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil;
+import
org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions;
+import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
+import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
+import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+
+/** Verifies the wide-table batch metadata scan (see FileLoaderUtils) reads
data correctly. */
+public class AlignedSeriesScanBatchTest extends AbstractAlignedSeriesScanTest {
+
+ @Test
+ public void testBatchScanReadsAlignedData() throws IOException,
IllegalPathException {
+ AlignedFullPath scanPath =
+ new AlignedFullPath(
+ TEST_DEVICE,
+ Arrays.asList("s1", "s2", "s3"),
Review Comment:
**[P2] The tests don't reach the case this PR targets**
This end-to-end test uses a 3-column, tree-style device, which fits in one
measurement leaf group. The cache tests mock the reader. Nothing covers:
- a real multi-leaf device (more than 256 series), which is the case being
optimized
- a table-model (V4) TsFile
- `ignoreNotExists=true` with absent columns (see the comment on
`TimeSeriesMetadataCache` L362)
- the planner wiring in
`DataNodeTableOperatorGenerator.computeTotalFieldColumnCount`. If that ever
returns 0, the feature silently turns off and every test still passes.
- an IT
As a reference, I wrote a throwaway test: write an aligned device with 700
INT32 columns via `TsFileWriter`, then compare `getDeviceTimeSeriesMetadata`
with the per-column `get()` for every sensor plus one absent sensor. It passed,
and something similar in the PR would be a cheap regression guard.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
Review Comment:
**[P3] Duplicated logic**
`isDevicePossiblyPresent` reimplements the BloomFilter block in `get()`. The
`try/finally` statistics block above is a verbatim copy of the one in `get()`.
The cache-enabled and cache-disabled branches of `getDeviceTimeSeriesMetadata`
also repeat the same "BloomFilter check, then read" sequence. Extracting shared
helpers, for example `recordLoadStatistics(...)`, `bloomFilterMayContain(...)`
and a single `loadDevice(reader, cacheEnabled)`, would keep the two paths from
drifting apart again.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
Review Comment:
**[P2] Design question: could this live in the existing `get()` miss path?**
This adds about 200 lines that run in parallel with `get()`: locking,
BloomFilter, placeholder caching and statistics. It then threads
`totalFieldColumnCount` from the planner through `SeriesScanOptions`,
`FileLoaderUtils` and the external scan util. The two paths have already
diverged twice (DEBUG logs, BloomFilter-first). They still differ in IO
accounting, per-measurement BloomFilter checks and hit-ratio stats.
The root cause is that each leaf-group miss walks the device index again. An
alternative: on a `get()` miss, walk the device index once and read every
measurement leaf group that overlaps `allSensors` (one TsFile call), then cache
them all. That would remove the ratio heuristic and the schema lookup, keep the
DEBUG, IO and BloomFilter behavior in one place, and also help scans this PR
doesn't touch, such as aggregation pushdown (`AggregationTableScanNode`),
tree-model scans and view scans.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
Review Comment:
**[P3] The device lock is held for the whole device read**
The per-(device, file) monitor is now held while the BloomFilter is checked
and every measurement leaf of the device is read and deserialized. The
per-column `get()` holds the same monitor for a single leaf. For devices with
more than 256 series, a concurrent narrow query on the same device and file
waits for the whole device read. This is probably acceptable, but it's worth
knowing about.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ QueryContext queryContext,
+ LongConsumer bloomFilterIoSizeRecorder)
+ throws IOException {
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+ false,
+ bloomFilterIoSizeRecorder,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+ false);
Review Comment:
**[P3] `externalTsFile` is not passed to the BloomFilter lookup**
`getDeviceTimeSeriesMetadata` uses `queryContext.isExternalTsFileScan()` to
choose the external reader, but this call always passes `externalTsFile =
false` to `BloomFilterCache.get`. For an external file that would cache its
BloomFilter under a `TsFileID` key that may collide with an internal file, and
open the external file through the internal reader table. The planner gate
currently makes this unreachable. Since this is a public method, could you pass
the flag through, or reject external scans up front with a `checkArgument`?
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java:
##########
@@ -375,38 +412,108 @@ private static AbstractAlignedTimeSeriesMetadata
loadAlignedTimeSeriesMetadataFr
} else {
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
- // if all the queried aligned sensors does not exist, we will return
null
- boolean exist = false;
for (String measurement : valueMeasurementList) {
- TimeseriesMetadata valueColumn =
+ valueTimeSeriesMetadataList.add(
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(),
deviceId, measurement),
allSensors,
- context.ignoreNotExistsDevice()
- || resource.getTimeIndexType() ==
ITimeIndex.FILE_TIME_INDEX_TYPE,
+ ignoreNotExists,
isDebug,
context,
- rootMeasurementMetadataIndexNodeOffset);
- exist = (exist || (valueColumn != null));
- valueTimeSeriesMetadataList.add(valueColumn);
- }
- if (!ignoreAllNullRows || exist) {
- alignedTimeSeriesMetadata =
- setModifications(
- resource,
- timeColumn,
- valueTimeSeriesMetadataList,
- alignedPath,
- context,
- globalTimeFilter,
- ignoreAllNullRows);
+ rootMeasurementMetadataIndexNodeOffset));
}
+ alignedTimeSeriesMetadata =
+ buildAlignedTimeSeriesMetadata(
+ resource,
+ alignedPath,
+ timeColumn,
+ valueTimeSeriesMetadataList,
+ context,
+ globalTimeFilter,
+ ignoreAllNullRows);
}
}
return alignedTimeSeriesMetadata;
}
+ static boolean shouldUseDeviceMetadataBatchScan(
+ int selectedColumnCount, int totalFieldColumnCount, boolean debug,
boolean offsetAddressed) {
+ // DEBUG queries keep the per-column cache's diagnostic logs;
offset-addressed external
+ // tsfiles cannot be resolved through the table metadata index; a
single-column device gains
+ // nothing from a batch read (one leaf either way)
+ return !debug
+ && !offsetAddressed
+ && totalFieldColumnCount > 1
Review Comment:
**[P2] The gate doesn't consider the index fan-out, so small devices gain
nothing**
`TsFileSequenceReader#readTimeseriesMetadata` reads the whole measurement
leaf group that contains the requested measurement. The writer starts a new
group every `max_degree_of_index_node` series (256 by default), and every
series in the group is deserialized with chunk metadata. `get()` then caches
every `allSensors` entry from that group. So for a device with 256 or fewer
series, the per-column path already does a single read for all columns. The
batch path can only save the extra device-index walks when the selected columns
span two or more leaf groups.
With `totalFieldColumnCount > 1`, a 10-field `SELECT *` still takes the
batch path and gets the costs described in the other comments without the
benefit. Consider also requiring `totalFieldColumnCount >
TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()`, or
gating on the number of leaf groups.
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java:
##########
@@ -72,6 +73,12 @@ public class FileLoaderUtils {
private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
SeriesScanCostMetricSet.getInstance();
+ /**
+ * Fraction of a table's field columns that a scan must select for it to use
the batch
+ * device-metadata read instead of the per-column path.
+ */
+ private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6;
Review Comment:
**[P2] Hard-coded threshold that doesn't match the PR description, and no
off switch**
The PR description still documents a configurable
`device_metadata_batch_scan_ratio` (default 0.8), an `IoTDBConfig` change, an
`ignoreNotExists` gate and `readDeviceMetadata`. None of these match the
current code. Please update it. Since 0.6 is hard-coded, if the batch path
misbehaves in production (e.g. the re-read case in the
`TimeSeriesMetadataCache` L362 comment), the only way to turn it off is a new
release. Could you keep a config switch, or include benchmark numbers that
justify 0.6?
##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java:
##########
@@ -280,4 +288,164 @@ public void testIgnoreNotExistsResultIsNotCached() throws
Exception {
.setMayCacheNonExistSeries(mayCacheNonExistSeries);
}
}
+
+ @Test
+ public void testGetDeviceTimeSeriesMetadata() throws Exception {
+ TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+ BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+ FileReaderManager fileReaderManager = FileReaderManager.getInstance();
+ TsFileID tsFileID = new TsFileID();
+ cache.clear();
+ bloomFilterCache.clear();
+ try {
+ String filePath = "target/batch-device.tsfile";
+ IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device");
+ Set<String> allSensors = new HashSet<>(Arrays.asList("", "s1", "s2"));
+
+ Statistics<?> timeStatistics =
Statistics.getStatsByType(TSDataType.INT64);
+ timeStatistics.setCount(1);
+ timeStatistics.setStartTime(0);
+ timeStatistics.setEndTime(10);
+ TimeseriesMetadata timeColumn =
+ new TimeseriesMetadata(
+ (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new
PublicBAOS());
+
+ Statistics<?> s1Statistics = Statistics.getStatsByType(TSDataType.INT32);
+ s1Statistics.setCount(1);
+ s1Statistics.setStartTime(0);
+ s1Statistics.setEndTime(10);
+ TimeseriesMetadata s1 =
+ new TimeseriesMetadata(
+ (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new
PublicBAOS());
+
+ // an empty series (count == 0) is mapped to null and cached as the null
placeholder
+ TimeseriesMetadata emptyS2 =
+ new TimeseriesMetadata(
+ (byte) 0,
+ 0,
+ "s2",
+ TSDataType.INT32,
+ Statistics.getStatsByType(TSDataType.INT32),
+ new PublicBAOS());
+
+ List<TimeseriesMetadata> deviceMetadata = new ArrayList<>();
+ deviceMetadata.add(timeColumn);
+ deviceMetadata.add(s1);
+ deviceMetadata.add(emptyS2);
+ // the batch read must carry chunk metadata, otherwise the downstream
chunk loader NPEs
+ s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata()));
+
+ TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class);
+ fileReaderManager.setReaderForTest(tsFileID, true, reader);
+
Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata);
+
+ Map<String, TimeseriesMetadata> result =
+ cache.getDeviceTimeSeriesMetadata(
+ filePath, tsFileID, deviceID, allSensors, new
QueryContext(false, false), false);
+
+ assertNotNull(result.get(""));
+ assertEquals("", result.get("").getMeasurementId());
+ assertNotNull(result.get("s1"));
+ assertEquals("s1", result.get("s1").getMeasurementId());
+ // chunk metadata must flow through to the result (and the cached copy)
unchanged
+ assertNotNull(result.get("s1").getChunkMetadataList());
+ assertNull(result.get("s2"));
+
+ // a second lookup of cached measurements must be served without
touching the reader again
+ Set<String> cachedSensors = new HashSet<>(Arrays.asList("", "s1"));
+ Map<String, TimeseriesMetadata> cachedResult =
+ cache.getDeviceTimeSeriesMetadata(
+ filePath, tsFileID, deviceID, cachedSensors, new
QueryContext(false, false), false);
+ assertNotNull(cachedResult.get("s1"));
+ Mockito.verify(reader,
Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID);
+
+ // the batch read also populated the per-measurement cache
+ TimeseriesMetadata cached =
+ cache.get(
+ filePath,
+ new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"),
+ Collections.singleton("s1"),
+ true,
+ false,
+ new QueryContext(false, false));
+ assertNotNull(cached);
+ assertEquals("s1", cached.getMeasurementId());
+ } finally {
+ cache.clear();
+ bloomFilterCache.clear();
+ fileReaderManager.closeFileAndRemoveReader(tsFileID);
+ }
+ }
+
+ @Test
+ public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws
Exception {
+ TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+ BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+ FileReaderManager fileReaderManager = FileReaderManager.getInstance();
+ TsFileID tsFileID = new TsFileID();
+ cache.clear();
+ bloomFilterCache.clear();
+ try {
+ String filePath = "target/batch-absent-device-bloom.tsfile";
+ IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent");
+ Set<String> allSensors = Collections.singleton("");
+
+ TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class);
+ fileReaderManager.setReaderForTest(tsFileID, true, reader);
+ // the bloom filter proves the device is missing, so the device must not
be read at all
+ Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class)))
+ .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1));
+
+ // an absent device yields an empty map, which
loadAlignedTimeSeriesMetadataBatch then
+ // distinguishes from "present but empty time column"
+ assertTrue(
+ cache
+ .getDeviceTimeSeriesMetadata(
+ filePath, tsFileID, deviceID, allSensors, new
QueryContext(false, false), false)
+ .isEmpty());
+ Mockito.verify(reader,
Mockito.never()).getDeviceTimeseriesMetadata(deviceID);
+ } finally {
+ cache.clear();
+ bloomFilterCache.clear();
+ fileReaderManager.closeFileAndRemoveReader(tsFileID);
+ }
+ }
+
+ @Test
+ public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws
Exception {
+ TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+ BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+ FileReaderManager fileReaderManager = FileReaderManager.getInstance();
+ TsFileID tsFileID = new TsFileID();
+ cache.clear();
+ bloomFilterCache.clear();
+ try {
+ String filePath = "target/batch-absent-device-throw.tsfile";
+ IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent");
+ Set<String> allSensors = Collections.singleton("");
+
+ TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class);
+ fileReaderManager.setReaderForTest(tsFileID, true, reader);
+ Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID))
+ .thenReturn(Collections.emptyList());
+ BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1);
+ bloomFilter.add(deviceID.toString() + ".");
+
Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter);
+
+ assertThrows(
+ IOException.class,
+ () ->
+ cache.getDeviceTimeSeriesMetadata(
+ filePath, tsFileID, deviceID, allSensors, new
QueryContext(false, false), false));
+
+ assertTrue(
+ cache
+ .getDeviceTimeSeriesMetadata(
+ filePath, tsFileID, deviceID, allSensors, new
QueryContext(false, false), true)
+ .isEmpty());
+ } finally {
Review Comment:
**[P3] Teardown doesn't clear the BloomFilter cache**
This test clears `BloomFilterCache` at the start but not in `finally`,
unlike the other tests in this class. A BloomFilter containing only
`root.absent.` stays cached under `TsFileID(-1, ...)`. Nothing breaks today
because every test here clears the cache at the start. A future test that
doesn't would see all its series filtered out depending on test order. Please
add `bloomFilterCache.clear();` to the `finally` block.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]