JackieTien97 commented on code in PR #19198:
URL: https://github.com/apache/iotdb/pull/19198#discussion_r4558365667


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {
+          cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, 
allSensors);
+          if (cached != null) {
+            return cached;
+          }
+          cacheHit = false;
+          long bloomFilterStartTime = System.nanoTime();
+          boolean devicePossiblyPresent =
+              isDevicePossiblyPresent(
+                  filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+          loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+          if (!devicePossiblyPresent) {
+            return Collections.emptyMap();
+          }
+          TsFileSequenceReader reader =
+              FileReaderManager.getInstance()
+                  .get(filePath, tsFileID, true, 
timeSeriesMetadataIoSizeRecorder, false);
+          return readDeviceTimeSeriesMetadata(
+              reader, tsFileID, device, allSensors, true, ignoreNotExists);
+        }
+      }
+      cacheHit = false;
+      long bloomFilterStartTime = System.nanoTime();
+      boolean devicePossiblyPresent =
+          isDevicePossiblyPresent(
+              filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+      loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+      if (!devicePossiblyPresent) {
+        return Collections.emptyMap();
+      }
+      TsFileSequenceReader reader =
+          FileReaderManager.getInstance()
+              .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, 
externalTsFile);
+      return readDeviceTimeSeriesMetadata(
+          reader, tsFileID, device, allSensors, false, ignoreNotExists);
+    } finally {
+      
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+      if (cacheHit) {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      } else {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      }
+    }
+  }
+
+  /**
+   * The time column (keyed by "") is written for every aligned device, so its 
bloom filter entry is
+   * a cheap per-file presence probe. A null filter cannot rule the device out.
+   */
+  private boolean isDevicePossiblyPresent(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      QueryContext queryContext,
+      LongConsumer bloomFilterIoSizeRecorder)
+      throws IOException {
+    BloomFilter bloomFilter =
+        BloomFilterCache.getInstance()
+            .get(
+                new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+                false,
+                bloomFilterIoSizeRecorder,
+                
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+                
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+                false);
+    return bloomFilter == null
+        || bloomFilter.contains(device.toString() + 
IoTDBConstant.PATH_SEPARATOR);
+  }
+
+  /**
+   * @return copies of the cached metadata for every measurement in {@code 
allSensors}, or null as
+   *     soon as one of them is not cached and the device therefore has to be 
read from disk
+   */
+  private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+      TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+    Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+    for (String measurement : allSensors) {
+      TimeseriesMetadata metadata =
+          lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, 
device, measurement));
+      if (metadata == null) {
+        return null;
+      }
+      result.put(
+          measurement,
+          metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new 
TimeseriesMetadata(metadata));
+    }
+    return result;
+  }
+
+  private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+      TsFileSequenceReader reader,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      boolean cacheEnabled,
+      boolean ignoreNotExists)
+      throws IOException {
+    // Use the chunk-metadata-including variant: the returned 
TimeseriesMetadata are passed to
+    // setModifications and then to DiskAlignedChunkMetadataLoader, which 
dereferences the chunk
+    // metadata list, so readDeviceMetadata (which skips chunk metadata) would 
NPE at query time.
+    List<TimeseriesMetadata> deviceMetadataList = 
reader.getDeviceTimeseriesMetadata(device);
+    if (deviceMetadataList.isEmpty()) {
+      // the caller already consulted the bloom filter, so the device could 
not be ruled out:
+      // missing metadata is an inconsistency unless ignoreNotExists is set
+      if (ignoreNotExists) {
+        return Collections.emptyMap();
+      }
+      throw new IOException(
+          
String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT, 
device));

Review Comment:
   **[P1] Wrong diagnostic for a device missing from the TsFile metadata**
   
   This path is reached when the resource's device-level time index says the 
device is in the file but the TsFile metadata does not contain it. 
`DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT` reads `%s does not exist in the 
resource file`. That constant is used by `RepairDataFileScanUtil` for the 
opposite case: a device that is in the TsFile but not in the resource. It also 
drops the file path. In a probe the cause message was `root.absent does not 
exist in the resource file`. The per-column path reports TsFile's `Device 
{root.absent} is not in tsFileMetaData of <file>`.
   
   Someone investigating the resulting `CorruptedTsFileException` would be sent 
to the `.resource` file. Could you add a dedicated constant to both the `en` 
and `zh` `StorageEngineMessages`, with the device and the TsFile path, e.g. 
`Device %s is recorded in the resource but missing from the metadata of TsFile 
%s`?



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);

Review Comment:
   **[P1] All-or-nothing cache check turns non-cacheable absent columns into a 
full device re-read on every query**
   
   `getCachedDeviceTimeSeriesMetadata` returns null as soon as one selected 
sensor is missing from the LRU. When a selected column is absent from the 
device in this file, `readDeviceTimeSeriesMetadata` only caches the 
`NULL_EXISTS_CACHE_PLACE_HOLDER` if `!ignoreNotExists && 
isMayCacheNonExistSeries()` is true. Otherwise the cache check fails on every 
query, and every query re-reads and re-deserializes the whole device, including 
the chunk metadata of every column. `ignoreNotExists` is true for files whose 
time index was degraded to `FILE_TIME_INDEX_TYPE` and for queries during 
startup recovery (`ignoreNotExistsDevice()`).
   
   The per-column `get()` handles the same case cheaply. Present columns stay 
cached, and absent columns are rejected by the cached per-measurement 
BloomFilter (`device.measurement`) without touching the file, barring false 
positives.
   
   I reproduced this with a mocked reader. The selected set is `{"", s1, s2}`, 
`s2` is absent from the file, and I ran 5 queries:
   - `ignoreNotExists=true`: batch `getDeviceTimeseriesMetadata` x5 vs 
per-column `readTimeseriesMetadata` x1
   - `ignoreNotExists=false`, `may_cache_nonexist_series=false`: x5 vs x1
   - defaults: x1 vs x1
   
   Sparse wide tables and columns added after older files were written make 
this common for `SELECT *`. A single evicted entry has a similar effect: it 
forces all leaf groups of the device to be re-read, while the per-column path 
re-reads only one.
   
   Possible fixes: when the cache check fails, probe the BloomFilter for the 
missing keys and treat filtered-out ones as known-absent. Alternatively, fetch 
only the missing measurements through the per-column `get()` and use the batch 
read only when many keys are missing.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {
+          cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, 
allSensors);
+          if (cached != null) {
+            return cached;
+          }
+          cacheHit = false;
+          long bloomFilterStartTime = System.nanoTime();
+          boolean devicePossiblyPresent =
+              isDevicePossiblyPresent(
+                  filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+          loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+          if (!devicePossiblyPresent) {
+            return Collections.emptyMap();
+          }
+          TsFileSequenceReader reader =
+              FileReaderManager.getInstance()
+                  .get(filePath, tsFileID, true, 
timeSeriesMetadataIoSizeRecorder, false);
+          return readDeviceTimeSeriesMetadata(
+              reader, tsFileID, device, allSensors, true, ignoreNotExists);
+        }
+      }
+      cacheHit = false;
+      long bloomFilterStartTime = System.nanoTime();
+      boolean devicePossiblyPresent =
+          isDevicePossiblyPresent(
+              filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+      loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+      if (!devicePossiblyPresent) {
+        return Collections.emptyMap();
+      }
+      TsFileSequenceReader reader =
+          FileReaderManager.getInstance()
+              .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, 
externalTsFile);
+      return readDeviceTimeSeriesMetadata(
+          reader, tsFileID, device, allSensors, false, ignoreNotExists);
+    } finally {
+      
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+      if (cacheHit) {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      } else {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      }
+    }
+  }
+
+  /**
+   * The time column (keyed by "") is written for every aligned device, so its 
bloom filter entry is
+   * a cheap per-file presence probe. A null filter cannot rule the device out.
+   */
+  private boolean isDevicePossiblyPresent(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      QueryContext queryContext,
+      LongConsumer bloomFilterIoSizeRecorder)
+      throws IOException {
+    BloomFilter bloomFilter =
+        BloomFilterCache.getInstance()
+            .get(
+                new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+                false,
+                bloomFilterIoSizeRecorder,
+                
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+                
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+                false);
+    return bloomFilter == null
+        || bloomFilter.contains(device.toString() + 
IoTDBConstant.PATH_SEPARATOR);
+  }
+
+  /**
+   * @return copies of the cached metadata for every measurement in {@code 
allSensors}, or null as
+   *     soon as one of them is not cached and the device therefore has to be 
read from disk
+   */
+  private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+      TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+    Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+    for (String measurement : allSensors) {
+      TimeseriesMetadata metadata =
+          lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, 
device, measurement));
+      if (metadata == null) {
+        return null;
+      }
+      result.put(
+          measurement,
+          metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new 
TimeseriesMetadata(metadata));
+    }
+    return result;
+  }
+
+  private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+      TsFileSequenceReader reader,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      boolean cacheEnabled,
+      boolean ignoreNotExists)
+      throws IOException {
+    // Use the chunk-metadata-including variant: the returned 
TimeseriesMetadata are passed to
+    // setModifications and then to DiskAlignedChunkMetadataLoader, which 
dereferences the chunk
+    // metadata list, so readDeviceMetadata (which skips chunk metadata) would 
NPE at query time.
+    List<TimeseriesMetadata> deviceMetadataList = 
reader.getDeviceTimeseriesMetadata(device);

Review Comment:
   **[P2] Metadata IO is not recorded, and load counts are now per device**
   
   `TsFileSequenceReader#getDeviceTimeseriesMetadata(IDeviceID)` has no IO-size 
recorder. Its internal `readData` calls don't record anything, and the 
`timeSeriesMetadataIoSizeRecorder` passed to `FileReaderManager.get` only 
covers opening the reader. So bytes read by the batch path never reach 
`loadTimeSeriesMetadataActualIOSize`.
   
   In addition, `LoadTimeSeriesMetadataFromCacheCount/FromDiskCount` now go up 
by 1 per device and file instead of 1 per series. For an 800-column device that 
is roughly 801 lookups versus 1. `AlignedSeriesScanBatchTest` asserts this new 
counting (`diskReadsBefore + 2`). As a result, `EXPLAIN ANALYZE` numbers for 
the same query change meaning depending on which side of the 0.6 threshold it 
falls.
   
   Could the batch read record its size? For example, sum the buffer sizes, or 
add a recorder-taking overload in TsFile. Please also either keep per-series 
counting or document the change.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {
+          cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, 
allSensors);

Review Comment:
   **[P3] Cache hit ratio is inflated on a miss**
   
   On a miss, `getCachedDeviceTimeSeriesMetadata` runs twice, once outside and 
once inside the lock. Each run calls `lruCache.getIfPresent` (the cache uses 
`recordStats()`) for every key it checks before the first missing one. Those 
hits are recorded twice, so `calculateTimeSeriesMetadataHitRatio()` reads high 
for wide scans. Checking presence once, or only on the second run, and copying 
entries only after every key has hit would avoid this.



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java:
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.storageengine.dataregion.read.reader.series;
+
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.AlignedFullPath;
+import 
org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil;
+import 
org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions;
+import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
+import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
+import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.read.common.block.TsBlock;
+import org.apache.tsfile.write.schema.MeasurementSchema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+
+/** Verifies the wide-table batch metadata scan (see FileLoaderUtils) reads 
data correctly. */
+public class AlignedSeriesScanBatchTest extends AbstractAlignedSeriesScanTest {
+
+  @Test
+  public void testBatchScanReadsAlignedData() throws IOException, 
IllegalPathException {
+    AlignedFullPath scanPath =
+        new AlignedFullPath(
+            TEST_DEVICE,
+            Arrays.asList("s1", "s2", "s3"),

Review Comment:
   **[P2] The tests don't reach the case this PR targets**
   
   This end-to-end test uses a 3-column, tree-style device, which fits in one 
measurement leaf group. The cache tests mock the reader. Nothing covers:
   - a real multi-leaf device (more than 256 series), which is the case being 
optimized
   - a table-model (V4) TsFile
   - `ignoreNotExists=true` with absent columns (see the comment on 
`TimeSeriesMetadataCache` L362)
   - the planner wiring in 
`DataNodeTableOperatorGenerator.computeTotalFieldColumnCount`. If that ever 
returns 0, the feature silently turns off and every test still passes.
   - an IT
   
   As a reference, I wrote a throwaway test: write an aligned device with 700 
INT32 columns via `TsFileWriter`, then compare `getDeviceTimeSeriesMetadata` 
with the per-column `get()` for every sensor plus one absent sensor. It passed, 
and something similar in the PR would be a cheap regression guard.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {
+          cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, 
allSensors);
+          if (cached != null) {
+            return cached;
+          }
+          cacheHit = false;
+          long bloomFilterStartTime = System.nanoTime();
+          boolean devicePossiblyPresent =
+              isDevicePossiblyPresent(
+                  filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+          loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+          if (!devicePossiblyPresent) {
+            return Collections.emptyMap();
+          }
+          TsFileSequenceReader reader =
+              FileReaderManager.getInstance()
+                  .get(filePath, tsFileID, true, 
timeSeriesMetadataIoSizeRecorder, false);
+          return readDeviceTimeSeriesMetadata(
+              reader, tsFileID, device, allSensors, true, ignoreNotExists);
+        }
+      }
+      cacheHit = false;
+      long bloomFilterStartTime = System.nanoTime();
+      boolean devicePossiblyPresent =
+          isDevicePossiblyPresent(
+              filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+      loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+      if (!devicePossiblyPresent) {
+        return Collections.emptyMap();
+      }
+      TsFileSequenceReader reader =
+          FileReaderManager.getInstance()
+              .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, 
externalTsFile);
+      return readDeviceTimeSeriesMetadata(
+          reader, tsFileID, device, allSensors, false, ignoreNotExists);
+    } finally {
+      
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+      if (cacheHit) {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      } else {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      }
+    }
+  }
+
+  /**
+   * The time column (keyed by "") is written for every aligned device, so its 
bloom filter entry is
+   * a cheap per-file presence probe. A null filter cannot rule the device out.
+   */
+  private boolean isDevicePossiblyPresent(

Review Comment:
   **[P3] Duplicated logic**
   
   `isDevicePossiblyPresent` reimplements the BloomFilter block in `get()`. The 
`try/finally` statistics block above is a verbatim copy of the one in `get()`. 
The cache-enabled and cache-disabled branches of `getDeviceTimeSeriesMetadata` 
also repeat the same "BloomFilter check, then read" sequence. Extracting shared 
helpers, for example `recordLoadStatistics(...)`, `bloomFilterMayContain(...)` 
and a single `loadDevice(reader, cacheEnabled)`, would keep the two paths from 
drifting apart again.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(

Review Comment:
   **[P2] Design question: could this live in the existing `get()` miss path?**
   
   This adds about 200 lines that run in parallel with `get()`: locking, 
BloomFilter, placeholder caching and statistics. It then threads 
`totalFieldColumnCount` from the planner through `SeriesScanOptions`, 
`FileLoaderUtils` and the external scan util. The two paths have already 
diverged twice (DEBUG logs, BloomFilter-first). They still differ in IO 
accounting, per-measurement BloomFilter checks and hit-ratio stats.
   
   The root cause is that each leaf-group miss walks the device index again. An 
alternative: on a `get()` miss, walk the device index once and read every 
measurement leaf group that overlaps `allSensors` (one TsFile call), then cache 
them all. That would remove the ratio heuristic and the schema lookup, keep the 
DEBUG, IO and BloomFilter behavior in one place, and also help scans this PR 
doesn't touch, such as aggregation pushdown (`AggregationTableScanNode`), 
tree-model scans and view scans.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {

Review Comment:
   **[P3] The device lock is held for the whole device read**
   
   The per-(device, file) monitor is now held while the BloomFilter is checked 
and every measurement leaf of the device is read and deserialized. The 
per-column `get()` holds the same monitor for a single leaf. For devices with 
more than 256 series, a concurrent narrow query on the same device and file 
waits for the whole device read. This is probably acceptable, but it's worth 
knowing about.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    long loadBloomFilterTime = 0;
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    LongConsumer bloomFilterIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {
+          cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, 
allSensors);
+          if (cached != null) {
+            return cached;
+          }
+          cacheHit = false;
+          long bloomFilterStartTime = System.nanoTime();
+          boolean devicePossiblyPresent =
+              isDevicePossiblyPresent(
+                  filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+          loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+          if (!devicePossiblyPresent) {
+            return Collections.emptyMap();
+          }
+          TsFileSequenceReader reader =
+              FileReaderManager.getInstance()
+                  .get(filePath, tsFileID, true, 
timeSeriesMetadataIoSizeRecorder, false);
+          return readDeviceTimeSeriesMetadata(
+              reader, tsFileID, device, allSensors, true, ignoreNotExists);
+        }
+      }
+      cacheHit = false;
+      long bloomFilterStartTime = System.nanoTime();
+      boolean devicePossiblyPresent =
+          isDevicePossiblyPresent(
+              filePath, tsFileID, device, queryContext, 
bloomFilterIoSizeRecorder);
+      loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+      if (!devicePossiblyPresent) {
+        return Collections.emptyMap();
+      }
+      TsFileSequenceReader reader =
+          FileReaderManager.getInstance()
+              .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, 
externalTsFile);
+      return readDeviceTimeSeriesMetadata(
+          reader, tsFileID, device, allSensors, false, ignoreNotExists);
+    } finally {
+      
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+      if (cacheHit) {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      } else {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskTime()
+            .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+      }
+    }
+  }
+
+  /**
+   * The time column (keyed by "") is written for every aligned device, so its 
bloom filter entry is
+   * a cheap per-file presence probe. A null filter cannot rule the device out.
+   */
+  private boolean isDevicePossiblyPresent(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      QueryContext queryContext,
+      LongConsumer bloomFilterIoSizeRecorder)
+      throws IOException {
+    BloomFilter bloomFilter =
+        BloomFilterCache.getInstance()
+            .get(
+                new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+                false,
+                bloomFilterIoSizeRecorder,
+                
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+                
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+                false);

Review Comment:
   **[P3] `externalTsFile` is not passed to the BloomFilter lookup**
   
   `getDeviceTimeSeriesMetadata` uses `queryContext.isExternalTsFileScan()` to 
choose the external reader, but this call always passes `externalTsFile = 
false` to `BloomFilterCache.get`. For an external file that would cache its 
BloomFilter under a `TsFileID` key that may collide with an internal file, and 
open the external file through the internal reader table. The planner gate 
currently makes this unreachable. Since this is a public method, could you pass 
the flag through, or reject external scans up front with a `checkArgument`?



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java:
##########
@@ -375,38 +412,108 @@ private static AbstractAlignedTimeSeriesMetadata 
loadAlignedTimeSeriesMetadataFr
       } else {
         List<TimeseriesMetadata> valueTimeSeriesMetadataList =
             new ArrayList<>(valueMeasurementList.size());
-        // if all the queried aligned sensors does not exist, we will return 
null
-        boolean exist = false;
         for (String measurement : valueMeasurementList) {
-          TimeseriesMetadata valueColumn =
+          valueTimeSeriesMetadataList.add(
               cache.get(
                   filePath,
                   new TimeSeriesMetadataCacheKey(resource.getTsFileID(), 
deviceId, measurement),
                   allSensors,
-                  context.ignoreNotExistsDevice()
-                      || resource.getTimeIndexType() == 
ITimeIndex.FILE_TIME_INDEX_TYPE,
+                  ignoreNotExists,
                   isDebug,
                   context,
-                  rootMeasurementMetadataIndexNodeOffset);
-          exist = (exist || (valueColumn != null));
-          valueTimeSeriesMetadataList.add(valueColumn);
-        }
-        if (!ignoreAllNullRows || exist) {
-          alignedTimeSeriesMetadata =
-              setModifications(
-                  resource,
-                  timeColumn,
-                  valueTimeSeriesMetadataList,
-                  alignedPath,
-                  context,
-                  globalTimeFilter,
-                  ignoreAllNullRows);
+                  rootMeasurementMetadataIndexNodeOffset));
         }
+        alignedTimeSeriesMetadata =
+            buildAlignedTimeSeriesMetadata(
+                resource,
+                alignedPath,
+                timeColumn,
+                valueTimeSeriesMetadataList,
+                context,
+                globalTimeFilter,
+                ignoreAllNullRows);
       }
     }
     return alignedTimeSeriesMetadata;
   }
 
+  static boolean shouldUseDeviceMetadataBatchScan(
+      int selectedColumnCount, int totalFieldColumnCount, boolean debug, 
boolean offsetAddressed) {
+    // DEBUG queries keep the per-column cache's diagnostic logs; 
offset-addressed external
+    // tsfiles cannot be resolved through the table metadata index; a 
single-column device gains
+    // nothing from a batch read (one leaf either way)
+    return !debug
+        && !offsetAddressed
+        && totalFieldColumnCount > 1

Review Comment:
   **[P2] The gate doesn't consider the index fan-out, so small devices gain 
nothing**
   
   `TsFileSequenceReader#readTimeseriesMetadata` reads the whole measurement 
leaf group that contains the requested measurement. The writer starts a new 
group every `max_degree_of_index_node` series (256 by default), and every 
series in the group is deserialized with chunk metadata. `get()` then caches 
every `allSensors` entry from that group. So for a device with 256 or fewer 
series, the per-column path already does a single read for all columns. The 
batch path can only save the extra device-index walks when the selected columns 
span two or more leaf groups.
   
   With `totalFieldColumnCount > 1`, a 10-field `SELECT *` still takes the 
batch path and gets the costs described in the other comments without the 
benefit. Consider also requiring `totalFieldColumnCount > 
TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()`, or 
gating on the number of leaf groups.



##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java:
##########
@@ -72,6 +73,12 @@ public class FileLoaderUtils {
   private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
       SeriesScanCostMetricSet.getInstance();
 
+  /**
+   * Fraction of a table's field columns that a scan must select for it to use 
the batch
+   * device-metadata read instead of the per-column path.
+   */
+  private static final double DEVICE_METADATA_BATCH_SCAN_RATIO = 0.6;

Review Comment:
   **[P2] Hard-coded threshold that doesn't match the PR description, and no 
off switch**
   
   The PR description still documents a configurable 
`device_metadata_batch_scan_ratio` (default 0.8), an `IoTDBConfig` change, an 
`ignoreNotExists` gate and `readDeviceMetadata`. None of these match the 
current code. Please update it. Since 0.6 is hard-coded, if the batch path 
misbehaves in production (e.g. the re-read case in the 
`TimeSeriesMetadataCache` L362 comment), the only way to turn it off is a new 
release. Could you keep a config switch, or include benchmark numbers that 
justify 0.6?



##########
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java:
##########
@@ -280,4 +288,164 @@ public void testIgnoreNotExistsResultIsNotCached() throws 
Exception {
           .setMayCacheNonExistSeries(mayCacheNonExistSeries);
     }
   }
+
+  @Test
+  public void testGetDeviceTimeSeriesMetadata() throws Exception {
+    TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+    BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+    FileReaderManager fileReaderManager = FileReaderManager.getInstance();
+    TsFileID tsFileID = new TsFileID();
+    cache.clear();
+    bloomFilterCache.clear();
+    try {
+      String filePath = "target/batch-device.tsfile";
+      IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device");
+      Set<String> allSensors = new HashSet<>(Arrays.asList("", "s1", "s2"));
+
+      Statistics<?> timeStatistics = 
Statistics.getStatsByType(TSDataType.INT64);
+      timeStatistics.setCount(1);
+      timeStatistics.setStartTime(0);
+      timeStatistics.setEndTime(10);
+      TimeseriesMetadata timeColumn =
+          new TimeseriesMetadata(
+              (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new 
PublicBAOS());
+
+      Statistics<?> s1Statistics = Statistics.getStatsByType(TSDataType.INT32);
+      s1Statistics.setCount(1);
+      s1Statistics.setStartTime(0);
+      s1Statistics.setEndTime(10);
+      TimeseriesMetadata s1 =
+          new TimeseriesMetadata(
+              (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new 
PublicBAOS());
+
+      // an empty series (count == 0) is mapped to null and cached as the null 
placeholder
+      TimeseriesMetadata emptyS2 =
+          new TimeseriesMetadata(
+              (byte) 0,
+              0,
+              "s2",
+              TSDataType.INT32,
+              Statistics.getStatsByType(TSDataType.INT32),
+              new PublicBAOS());
+
+      List<TimeseriesMetadata> deviceMetadata = new ArrayList<>();
+      deviceMetadata.add(timeColumn);
+      deviceMetadata.add(s1);
+      deviceMetadata.add(emptyS2);
+      // the batch read must carry chunk metadata, otherwise the downstream 
chunk loader NPEs
+      s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata()));
+
+      TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class);
+      fileReaderManager.setReaderForTest(tsFileID, true, reader);
+      
Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID)).thenReturn(deviceMetadata);
+
+      Map<String, TimeseriesMetadata> result =
+          cache.getDeviceTimeSeriesMetadata(
+              filePath, tsFileID, deviceID, allSensors, new 
QueryContext(false, false), false);
+
+      assertNotNull(result.get(""));
+      assertEquals("", result.get("").getMeasurementId());
+      assertNotNull(result.get("s1"));
+      assertEquals("s1", result.get("s1").getMeasurementId());
+      // chunk metadata must flow through to the result (and the cached copy) 
unchanged
+      assertNotNull(result.get("s1").getChunkMetadataList());
+      assertNull(result.get("s2"));
+
+      // a second lookup of cached measurements must be served without 
touching the reader again
+      Set<String> cachedSensors = new HashSet<>(Arrays.asList("", "s1"));
+      Map<String, TimeseriesMetadata> cachedResult =
+          cache.getDeviceTimeSeriesMetadata(
+              filePath, tsFileID, deviceID, cachedSensors, new 
QueryContext(false, false), false);
+      assertNotNull(cachedResult.get("s1"));
+      Mockito.verify(reader, 
Mockito.times(1)).getDeviceTimeseriesMetadata(deviceID);
+
+      // the batch read also populated the per-measurement cache
+      TimeseriesMetadata cached =
+          cache.get(
+              filePath,
+              new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"),
+              Collections.singleton("s1"),
+              true,
+              false,
+              new QueryContext(false, false));
+      assertNotNull(cached);
+      assertEquals("s1", cached.getMeasurementId());
+    } finally {
+      cache.clear();
+      bloomFilterCache.clear();
+      fileReaderManager.closeFileAndRemoveReader(tsFileID);
+    }
+  }
+
+  @Test
+  public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws 
Exception {
+    TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+    BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+    FileReaderManager fileReaderManager = FileReaderManager.getInstance();
+    TsFileID tsFileID = new TsFileID();
+    cache.clear();
+    bloomFilterCache.clear();
+    try {
+      String filePath = "target/batch-absent-device-bloom.tsfile";
+      IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent");
+      Set<String> allSensors = Collections.singleton("");
+
+      TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class);
+      fileReaderManager.setReaderForTest(tsFileID, true, reader);
+      // the bloom filter proves the device is missing, so the device must not 
be read at all
+      Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class)))
+          .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1));
+
+      // an absent device yields an empty map, which 
loadAlignedTimeSeriesMetadataBatch then
+      // distinguishes from "present but empty time column"
+      assertTrue(
+          cache
+              .getDeviceTimeSeriesMetadata(
+                  filePath, tsFileID, deviceID, allSensors, new 
QueryContext(false, false), false)
+              .isEmpty());
+      Mockito.verify(reader, 
Mockito.never()).getDeviceTimeseriesMetadata(deviceID);
+    } finally {
+      cache.clear();
+      bloomFilterCache.clear();
+      fileReaderManager.closeFileAndRemoveReader(tsFileID);
+    }
+  }
+
+  @Test
+  public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws 
Exception {
+    TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
+    BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance();
+    FileReaderManager fileReaderManager = FileReaderManager.getInstance();
+    TsFileID tsFileID = new TsFileID();
+    cache.clear();
+    bloomFilterCache.clear();
+    try {
+      String filePath = "target/batch-absent-device-throw.tsfile";
+      IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent");
+      Set<String> allSensors = Collections.singleton("");
+
+      TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class);
+      fileReaderManager.setReaderForTest(tsFileID, true, reader);
+      Mockito.when(reader.getDeviceTimeseriesMetadata(deviceID))
+          .thenReturn(Collections.emptyList());
+      BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1);
+      bloomFilter.add(deviceID.toString() + ".");
+      
Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter);
+
+      assertThrows(
+          IOException.class,
+          () ->
+              cache.getDeviceTimeSeriesMetadata(
+                  filePath, tsFileID, deviceID, allSensors, new 
QueryContext(false, false), false));
+
+      assertTrue(
+          cache
+              .getDeviceTimeSeriesMetadata(
+                  filePath, tsFileID, deviceID, allSensors, new 
QueryContext(false, false), true)
+              .isEmpty());
+    } finally {

Review Comment:
   **[P3] Teardown doesn't clear the BloomFilter cache**
   
   This test clears `BloomFilterCache` at the start but not in `finally`, 
unlike the other tests in this class. A BloomFilter containing only 
`root.absent.` stays cached under `TsFileID(-1, ...)`. Nothing breaks today 
because every test here clears the cache at the start. A future test that 
doesn't would see all its series filtered out depending on test order. Please 
add `bloomFilterCache.clear();` to the `finally` block.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to