ColinLeeo commented on code in PR #19198:
URL: https://github.com/apache/iotdb/pull/19198#discussion_r4549606237


##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,177 @@ public TimeseriesMetadata get(
     }
   }
 
+  /**
+   * Load the TimeseriesMetadata of {@code allSensors} of a device, reading 
the device in a single
+   * batch when the per-measurement cache does not already hold all of them. 
This is used by
+   * wide-table scans (a device with many columns) to avoid walking the device 
metadata index once
+   * per column / leaf node.
+   *
+   * <p>The returned map always holds copies of the cached (or freshly read) 
metadata, so the caller
+   * may apply {@code setModified} and {@code setChunkMetadataLoader} without 
corrupting cached
+   * entries — matching {@link #get}.
+   *
+   * @param filePath path of the tsfile
+   * @param tsFileID id of the tsfile
+   * @param device target device
+   * @param allSensors measurements the caller needs, including the time 
column keyed by ""
+   * @param queryContext query context used for metrics and to detect external 
tsfiles
+   * @param ignoreNotExists whether a device missing from a file that claims a 
device-level time
+   *     index should be skipped instead of reported as an inconsistency
+   * @return map from measurementId (including the time column, keyed by "") 
to TimeseriesMetadata;
+   *     entries whose series is empty (count == 0) are mapped to null
+   * @throws IOException if the metadata cannot be read from disk or the 
device is unexpectedly
+   *     absent
+   */
+  public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+      String filePath,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      QueryContext queryContext,
+      boolean ignoreNotExists)
+      throws IOException {
+    long startTime = System.nanoTime();
+    boolean externalTsFile = queryContext.isExternalTsFileScan();
+    LongConsumer timeSeriesMetadataIoSizeRecorder =
+        
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+    boolean cacheHit = true;
+
+    try {
+      if (CACHE_ENABLE && !externalTsFile) {
+        Map<String, TimeseriesMetadata> cached =
+            getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+        if (cached != null) {
+          return cached;
+        }
+        // serialize concurrent reads of the same device+file (same lock key 
as get) so threads
+        // don't each deserialize the whole device and race on the shared 
reader
+        synchronized (
+            devices.computeIfAbsent(device.toString() + SEPARATOR + filePath, 
WeakReference::new)) {
+          cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device, 
allSensors);
+          if (cached != null) {
+            return cached;
+          }
+          cacheHit = false;
+          TsFileSequenceReader reader =
+              FileReaderManager.getInstance()
+                  .get(filePath, tsFileID, true, 
timeSeriesMetadataIoSizeRecorder, false);
+          return readDeviceTimeSeriesMetadata(
+              reader, tsFileID, device, allSensors, true, ignoreNotExists);
+        }
+      }
+      cacheHit = false;
+      TsFileSequenceReader reader =
+          FileReaderManager.getInstance()
+              .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, 
externalTsFile);
+      return readDeviceTimeSeriesMetadata(
+          reader, tsFileID, device, allSensors, false, ignoreNotExists);
+    } finally {
+      if (cacheHit) {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromCacheTime()
+            .getAndAdd(System.nanoTime() - startTime);
+      } else {
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskCount()
+            .incrementAndGet();
+        queryContext
+            .getQueryStatistics()
+            .getLoadTimeSeriesMetadataFromDiskTime()
+            .getAndAdd(System.nanoTime() - startTime);
+      }
+    }
+  }
+
+  /**
+   * @return copies of the cached metadata for every measurement in {@code 
allSensors}, or null as
+   *     soon as one of them is not cached and the device therefore has to be 
read from disk
+   */
+  private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+      TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+    Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+    for (String measurement : allSensors) {
+      TimeseriesMetadata metadata =
+          lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, 
device, measurement));
+      if (metadata == null) {
+        return null;
+      }
+      result.put(
+          measurement,
+          metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new 
TimeseriesMetadata(metadata));
+    }
+    return result;
+  }
+
+  private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+      TsFileSequenceReader reader,
+      TsFileID tsFileID,
+      IDeviceID device,
+      Set<String> allSensors,
+      boolean cacheEnabled,
+      boolean ignoreNotExists)
+      throws IOException {
+    // Use the chunk-metadata-including variant: the returned 
TimeseriesMetadata are passed to
+    // setModifications and then to DiskAlignedChunkMetadataLoader, which 
dereferences the chunk
+    // metadata list, so readDeviceMetadata (which skips chunk metadata) would 
NPE at query time.
+    List<TimeseriesMetadata> deviceMetadataList = 
reader.getDeviceTimeseriesMetadata(device);

Review Comment:
   Added the cached time-column bloom filter probe before 
`reader.getDeviceTimeseriesMetadata`: an absent device now returns an empty map 
without the metadata-index read, the missing-device skip/throw rules are 
unchanged, and the bloom filter cache/disk/time statistics are accounted the 
same way as the per-column `get()`. The tests also assert the device is never 
read when the filter excludes it.
   _🤖 Addressed by [Claude Code](https://claude.com/claude-code)_



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to