ColinLeeo commented on code in PR #19198:
URL: https://github.com/apache/iotdb/pull/19198#discussion_r4559030502
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
Review Comment:
Cached measurements are now reused individually and the remaining ones are
checked against the per-measurement bloom filter before any device read, so
absent non-cacheable columns no longer trigger a whole-device re-read.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ QueryContext queryContext,
+ LongConsumer bloomFilterIoSizeRecorder)
+ throws IOException {
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+ false,
+ bloomFilterIoSizeRecorder,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+ false);
+ return bloomFilter == null
+ || bloomFilter.contains(device.toString() +
IoTDBConstant.PATH_SEPARATOR);
+ }
+
+ /**
+ * @return copies of the cached metadata for every measurement in {@code
allSensors}, or null as
+ * soon as one of them is not cached and the device therefore has to be
read from disk
+ */
+ private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+ TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ for (String measurement : allSensors) {
+ TimeseriesMetadata metadata =
+ lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID,
device, measurement));
+ if (metadata == null) {
+ return null;
+ }
+ result.put(
+ measurement,
+ metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new
TimeseriesMetadata(metadata));
+ }
+ return result;
+ }
+
+ private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+ TsFileSequenceReader reader,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ boolean cacheEnabled,
+ boolean ignoreNotExists)
+ throws IOException {
+ // Use the chunk-metadata-including variant: the returned
TimeseriesMetadata are passed to
+ // setModifications and then to DiskAlignedChunkMetadataLoader, which
dereferences the chunk
+ // metadata list, so readDeviceMetadata (which skips chunk metadata) would
NPE at query time.
+ List<TimeseriesMetadata> deviceMetadataList =
reader.getDeviceTimeseriesMetadata(device);
+ if (deviceMetadataList.isEmpty()) {
+ // the caller already consulted the bloom filter, so the device could
not be ruled out:
+ // missing metadata is an inconsistency unless ignoreNotExists is set
+ if (ignoreNotExists) {
+ return Collections.emptyMap();
+ }
+ throw new IOException(
+
String.format(StorageEngineMessages.DEVICE_DOES_NOT_EXIST_IN_RESOURCE_FILE_FMT,
device));
Review Comment:
Added
`EXCEPTION_DEVICE_ARG_IS_RECORDED_IN_THE_RESOURCE_BUT_MISSING_FROM_THE_METADATA_OF_TSFILE_ARG_DE00C374`
to the en/zh `StorageEngineMessages`, naming the device and the TsFile path;
the old constant stays for its original use.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ QueryContext queryContext,
+ LongConsumer bloomFilterIoSizeRecorder)
+ throws IOException {
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+ false,
+ bloomFilterIoSizeRecorder,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+ false);
+ return bloomFilter == null
+ || bloomFilter.contains(device.toString() +
IoTDBConstant.PATH_SEPARATOR);
+ }
+
+ /**
+ * @return copies of the cached metadata for every measurement in {@code
allSensors}, or null as
+ * soon as one of them is not cached and the device therefore has to be
read from disk
+ */
+ private Map<String, TimeseriesMetadata> getCachedDeviceTimeSeriesMetadata(
+ TsFileID tsFileID, IDeviceID device, Set<String> allSensors) {
+ Map<String, TimeseriesMetadata> result = new HashMap<>(allSensors.size());
+ for (String measurement : allSensors) {
+ TimeseriesMetadata metadata =
+ lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID,
device, measurement));
+ if (metadata == null) {
+ return null;
+ }
+ result.put(
+ measurement,
+ metadata == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new
TimeseriesMetadata(metadata));
+ }
+ return result;
+ }
+
+ private Map<String, TimeseriesMetadata> readDeviceTimeSeriesMetadata(
+ TsFileSequenceReader reader,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ boolean cacheEnabled,
+ boolean ignoreNotExists)
+ throws IOException {
+ // Use the chunk-metadata-including variant: the returned
TimeseriesMetadata are passed to
+ // setModifications and then to DiskAlignedChunkMetadataLoader, which
dereferences the chunk
+ // metadata list, so readDeviceMetadata (which skips chunk metadata) would
NPE at query time.
+ List<TimeseriesMetadata> deviceMetadataList =
reader.getDeviceTimeseriesMetadata(device);
Review Comment:
Load counts are now incremented per series (cache hits and disk lookups) so
EXPLAIN ANALYZE stays comparable; the batch read's byte count is documented as
not reportable because `getDeviceTimeseriesMetadata` has no IO-size recorder (a
TsFile overload is a follow-up).
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
Review Comment:
Changed the check to one pass over all keys outside the lock and a second
pass only over the missing keys, so cached hits are recorded once instead of
twice.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java:
##########
@@ -316,6 +317,216 @@ public TimeseriesMetadata get(
}
}
+ /**
+ * Load the TimeseriesMetadata of {@code allSensors} of a device, reading
the device in a single
+ * batch when the per-measurement cache does not already hold all of them.
This is used by
+ * wide-table scans (a device with many columns) to avoid walking the device
metadata index once
+ * per column / leaf node.
+ *
+ * <p>The returned map always holds copies of the cached (or freshly read)
metadata, so the caller
+ * may apply {@code setModified} and {@code setChunkMetadataLoader} without
corrupting cached
+ * entries — matching {@link #get}.
+ *
+ * @param filePath path of the tsfile
+ * @param tsFileID id of the tsfile
+ * @param device target device
+ * @param allSensors measurements the caller needs, including the time
column keyed by ""
+ * @param queryContext query context used for metrics and to detect external
tsfiles
+ * @param ignoreNotExists whether a device missing from a file that claims a
device-level time
+ * index should be skipped instead of reported as an inconsistency
+ * @return map from measurementId (including the time column, keyed by "")
to TimeseriesMetadata;
+ * entries whose series is empty (count == 0) are mapped to null
+ * @throws IOException if the metadata cannot be read from disk or the
device is unexpectedly
+ * absent
+ */
+ public Map<String, TimeseriesMetadata> getDeviceTimeSeriesMetadata(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ Set<String> allSensors,
+ QueryContext queryContext,
+ boolean ignoreNotExists)
+ throws IOException {
+ long startTime = System.nanoTime();
+ long loadBloomFilterTime = 0;
+ boolean externalTsFile = queryContext.isExternalTsFileScan();
+ LongConsumer timeSeriesMetadataIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet;
+ LongConsumer bloomFilterIoSizeRecorder =
+
queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet;
+ boolean cacheHit = true;
+
+ try {
+ if (CACHE_ENABLE && !externalTsFile) {
+ Map<String, TimeseriesMetadata> cached =
+ getCachedDeviceTimeSeriesMetadata(tsFileID, device, allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ // serialize concurrent reads of the same device+file (same lock key
as get) so threads
+ // don't each deserialize the whole device and race on the shared
reader
+ synchronized (
+ devices.computeIfAbsent(device.toString() + SEPARATOR + filePath,
WeakReference::new)) {
+ cached = getCachedDeviceTimeSeriesMetadata(tsFileID, device,
allSensors);
+ if (cached != null) {
+ return cached;
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true,
timeSeriesMetadataIoSizeRecorder, false);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, true, ignoreNotExists);
+ }
+ }
+ cacheHit = false;
+ long bloomFilterStartTime = System.nanoTime();
+ boolean devicePossiblyPresent =
+ isDevicePossiblyPresent(
+ filePath, tsFileID, device, queryContext,
bloomFilterIoSizeRecorder);
+ loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime;
+ if (!devicePossiblyPresent) {
+ return Collections.emptyMap();
+ }
+ TsFileSequenceReader reader =
+ FileReaderManager.getInstance()
+ .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder,
externalTsFile);
+ return readDeviceTimeSeriesMetadata(
+ reader, tsFileID, device, allSensors, false, ignoreNotExists);
+ } finally {
+
queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime);
+ if (cacheHit) {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ } else {
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskCount()
+ .incrementAndGet();
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
+ }
+ }
+ }
+
+ /**
+ * The time column (keyed by "") is written for every aligned device, so its
bloom filter entry is
+ * a cheap per-file presence probe. A null filter cannot rule the device out.
+ */
+ private boolean isDevicePossiblyPresent(
+ String filePath,
+ TsFileID tsFileID,
+ IDeviceID device,
+ QueryContext queryContext,
+ LongConsumer bloomFilterIoSizeRecorder)
+ throws IOException {
+ BloomFilter bloomFilter =
+ BloomFilterCache.getInstance()
+ .get(
+ new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID),
+ false,
+ bloomFilterIoSizeRecorder,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet,
+
queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet,
+ false);
Review Comment:
`externalTsFile` is now passed through to the `BloomFilterCache.get` call;
the planner still keeps external scans on the per-column path.
_🤖 Addressed by [Claude Code](https://claude.com/claude-code)_
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]