This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a8e5080fc9c Remove useless metrics in AggTableScan and make the
TimeseriesMetadataCache get time as FI level
a8e5080fc9c is described below
commit a8e5080fc9c785efbe5d8e8d4d3b8eef6023203b
Author: Beyyes <[email protected]>
AuthorDate: Mon Mar 17 10:26:46 2025 +0800
Remove useless metrics in AggTableScan and make the TimeseriesMetadataCache
get time as FI level
---
.../fragment/FragmentInstanceContext.java | 4 ++-
.../execution/fragment/QueryStatistics.java | 10 ++++++
.../relational/aggregation/TableAggregator.java | 9 +----
.../metric/SeriesScanCostMetricSet.java | 40 +++++++++++++++++++++-
.../buffer/TimeSeriesMetadataCache.java | 20 +++++------
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
6 files changed, 62 insertions(+), 22 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index f467ee14c77..561eb432c6a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -766,7 +766,9 @@ public class FragmentInstanceContext extends QueryContext {
.recordTimeSeriesMetadataMetrics(
getQueryStatistics().getLoadTimeSeriesMetadataFromCacheCount().get(),
getQueryStatistics().getLoadTimeSeriesMetadataFromDiskCount().get(),
-
getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize().get());
+ getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize().get(),
+
getQueryStatistics().getLoadTimeSeriesMetadataFromCacheTime().get(),
+
getQueryStatistics().getLoadTimeSeriesMetadataFromDiskTime().get());
SeriesScanCostMetricSet.getInstance()
.recordConstructChunkReadersCount(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
index b1ddfdbf4d7..277e9106520 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryStatistics.java
@@ -54,7 +54,9 @@ public class QueryStatistics {
private final AtomicLong loadTimeSeriesMetadataAlignedMemUnSeqTime = new
AtomicLong(0);
private final AtomicLong loadTimeSeriesMetadataFromCacheCount = new
AtomicLong(0);
+ private final AtomicLong loadTimeSeriesMetadataFromCacheTime = new
AtomicLong(0);
private final AtomicLong loadTimeSeriesMetadataFromDiskCount = new
AtomicLong(0);
+ private final AtomicLong loadTimeSeriesMetadataFromDiskTime = new
AtomicLong(0);
private final AtomicLong loadTimeSeriesMetadataActualIOSize = new
AtomicLong(0);
// statistics for count and time of construct chunk readers(disk io and
decompress)
@@ -274,10 +276,18 @@ public class QueryStatistics {
return loadTimeSeriesMetadataFromCacheCount;
}
+ public AtomicLong getLoadTimeSeriesMetadataFromCacheTime() {
+ return loadTimeSeriesMetadataFromCacheTime;
+ }
+
public AtomicLong getLoadTimeSeriesMetadataFromDiskCount() {
return loadTimeSeriesMetadataFromDiskCount;
}
+ public AtomicLong getLoadTimeSeriesMetadataFromDiskTime() {
+ return loadTimeSeriesMetadataFromDiskTime;
+ }
+
public TQueryStatistics toThrift() {
return new TQueryStatistics(
loadTimeSeriesMetadataDiskSeqCount.get(),
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableAggregator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableAggregator.java
index fbbc7083127..955532cea6d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableAggregator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/TableAggregator.java
@@ -37,7 +37,6 @@ import static java.util.Objects.requireNonNull;
import static
org.apache.iotdb.db.queryengine.execution.aggregation.TreeAggregator.QUERY_EXECUTION_METRICS;
import static
org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.TIME_COLUMN_TEMPLATE;
import static
org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.AGGREGATION_FROM_RAW_DATA;
-import static
org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.AGGREGATION_FROM_STATISTICS;
public class TableAggregator {
private final TableAccumulator accumulator;
@@ -106,13 +105,7 @@ public class TableAggregator {
}
public void processStatistics(Statistics[] statistics) {
- long startTime = System.nanoTime();
- try {
- accumulator.addStatistics(statistics);
- } finally {
- QUERY_EXECUTION_METRICS.recordExecutionCost(
- AGGREGATION_FROM_STATISTICS, System.nanoTime() - startTime);
- }
+ accumulator.addStatistics(statistics);
}
public boolean hasFinalResult() {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
index ebaa7c030d9..bd006dca7ef 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
@@ -179,6 +179,8 @@ public class SeriesScanCostMetricSet implements IMetricSet {
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
private Counter loadTimeSeriesMetadataActualIOSizeCounter =
DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private Timer loadTimeSeriesMetadataFromCacheTimer =
DoNothingMetricManager.DO_NOTHING_TIMER;
+ private Timer loadTimeSeriesMetadataFromDiskTimer =
DoNothingMetricManager.DO_NOTHING_TIMER;
public void recordNonAlignedTimeSeriesMetadataCount(long c1, long c2, long
c3, long c4) {
loadTimeSeriesMetadataDiskSeqHistogram.update(c1);
@@ -211,10 +213,14 @@ public class SeriesScanCostMetricSet implements
IMetricSet {
public void recordTimeSeriesMetadataMetrics(
long loadTimeSeriesMetadataFromCacheCount,
long loadTimeSeriesMetadataFromDiskCount,
- long loadTimeSeriesMetadataActualIOSize) {
+ long loadTimeSeriesMetadataActualIOSize,
+ long loadTimeSeriesMetadataFromCacheTime,
+ long loadTimeSeriesMetadataFromDiskTime) {
loadTimeSeriesMetadataFromCacheCountHistogram.update(loadTimeSeriesMetadataFromCacheCount);
loadTimeSeriesMetadataFromDiskCountHistogram.update(loadTimeSeriesMetadataFromDiskCount);
loadTimeSeriesMetadataActualIOSizeCounter.inc(loadTimeSeriesMetadataActualIOSize);
+
loadTimeSeriesMetadataFromCacheTimer.updateNanos(loadTimeSeriesMetadataFromCacheTime);
+
loadTimeSeriesMetadataFromDiskTimer.updateNanos(loadTimeSeriesMetadataFromDiskTime);
}
private void bindTimeseriesMetadata(AbstractMetricService metricService) {
@@ -408,12 +414,30 @@ public class SeriesScanCostMetricSet implements
IMetricSet {
MetricLevel.IMPORTANT,
Tag.TYPE.toString(),
TIMESERIES_METADATA);
+ loadTimeSeriesMetadataFromCacheTimer =
+ metricService.getOrCreateTimer(
+ Metric.METRIC_QUERY_CACHE_TIMER.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.TYPE.toString(),
+ TIMESERIES_METADATA,
+ Tag.FROM.toString(),
+ CACHE);
+ loadTimeSeriesMetadataFromDiskTimer =
+ metricService.getOrCreateTimer(
+ Metric.METRIC_QUERY_CACHE_TIMER.toString(),
+ MetricLevel.IMPORTANT,
+ Tag.TYPE.toString(),
+ TIMESERIES_METADATA,
+ Tag.FROM.toString(),
+ DISK);
}
private void unbindTimeSeriesMetadataCache(AbstractMetricService
metricService) {
loadTimeSeriesMetadataFromCacheCountHistogram =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
loadTimeSeriesMetadataFromDiskCountHistogram =
DoNothingMetricManager.DO_NOTHING_HISTOGRAM;
loadTimeSeriesMetadataActualIOSizeCounter =
DoNothingMetricManager.DO_NOTHING_COUNTER;
+ loadTimeSeriesMetadataFromCacheTimer =
DoNothingMetricManager.DO_NOTHING_TIMER;
+ loadTimeSeriesMetadataFromDiskTimer =
DoNothingMetricManager.DO_NOTHING_TIMER;
metricService.remove(
MetricType.HISTOGRAM,
@@ -434,6 +458,20 @@ public class SeriesScanCostMetricSet implements IMetricSet
{
Metric.QUERY_DISK_READ.toString(),
Tag.TYPE.toString(),
TIMESERIES_METADATA);
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.METRIC_QUERY_CACHE_TIMER.toString(),
+ Tag.TYPE.toString(),
+ TIMESERIES_METADATA,
+ Tag.FROM.toString(),
+ CACHE);
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.METRIC_QUERY_CACHE_TIMER.toString(),
+ Tag.TYPE.toString(),
+ TIMESERIES_METADATA,
+ Tag.FROM.toString(),
+ DISK);
}
private void unbindTimeseriesMetadata(AbstractMetricService metricService) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
index 9b4bb17f63c..3fb377fabba 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
@@ -27,7 +27,6 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.DataNodeMemoryConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
-import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
@@ -55,8 +54,6 @@ import java.util.WeakHashMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.LongConsumer;
-import static
org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_CACHE;
-import static
org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_TIMESERIES_METADATA_FILE;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfCharArray;
/**
@@ -73,9 +70,6 @@ public class TimeSeriesMetadataCache {
private static final IMemoryBlock CACHE_MEMORY_BLOCK;
private static final boolean CACHE_ENABLE =
memoryConfig.isMetaDataCacheEnable();
- private static final SeriesScanCostMetricSet SERIES_SCAN_COST_METRIC_SET =
- SeriesScanCostMetricSet.getInstance();
-
private final Cache<TimeSeriesMetadataCacheKey, TimeseriesMetadata> lruCache;
private final AtomicLong entryAverageSize = new AtomicLong(0);
@@ -244,17 +238,19 @@ public class TimeSeriesMetadataCache {
.getQueryStatistics()
.getLoadTimeSeriesMetadataFromCacheCount()
.incrementAndGet();
- // in metric panel, loading BloomFilter time is included in loading
TimeSeriesMetadata
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- READ_TIMESERIES_METADATA_CACHE, System.nanoTime() - startTime);
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromCacheTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
} else {
queryContext
.getQueryStatistics()
.getLoadTimeSeriesMetadataFromDiskCount()
.incrementAndGet();
- // in metric panel, loading BloomFilter time is included in loading
TimeSeriesMetadata
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- READ_TIMESERIES_METADATA_FILE, System.nanoTime() - startTime);
+ queryContext
+ .getQueryStatistics()
+ .getLoadTimeSeriesMetadataFromDiskTime()
+ .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime);
}
}
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index c1f94c76e38..554f7d6226b 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -111,6 +111,7 @@ public enum Metric {
MEMORY_USAGE_MONITOR("memory_usage_monitor"),
METRIC_LOAD_TIME_SERIES_METADATA("metric_load_time_series_metadata"),
METRIC_QUERY_CACHE("metric_query_cache"),
+ METRIC_QUERY_CACHE_TIMER("metric_query_cache_timer"),
QUERY_METADATA_COST("query_metadata_cost"),
DISPATCHER("dispatcher"),
QUERY_EXECUTION("query_execution"),
