This is an automated email from the ASF dual-hosted git repository. caogaofei pushed a commit to branch beyyes/query_metrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8fe5ce18a85c2be32e823fa64ef0bc736748aa07 Author: Beyyes <[email protected]> AuthorDate: Tue Dec 12 22:24:00 2023 +0800 tmp --- .../metric/QueryExecutionMetricSet.java | 24 +++++------ .../queryengine/metric/QueryPlanCostMetricSet.java | 10 +++-- .../metric/QueryRelatedResourceMetricSet.java | 50 ++++++++++++++++++++-- .../queryengine/plan/execution/QueryExecution.java | 6 +++ .../iotdb/commons/service/metric/enums/Metric.java | 1 + 5 files changed, 72 insertions(+), 19 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java index 91d7857aa58..ed493ab738b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryExecutionMetricSet.java @@ -64,10 +64,9 @@ public class QueryExecutionMetricSet implements IMetricSet { dispatchReadTimer = DoNothingMetricManager.DO_NOTHING_TIMER; Arrays.asList(WAIT_FOR_DISPATCH, DISPATCH_READ) .forEach( - stage -> { - metricService.remove( - MetricType.TIMER, Metric.DISPATCHER.toString(), Tag.STAGE.toString(), stage); - }); + stage -> + metricService.remove( + MetricType.TIMER, Metric.DISPATCHER.toString(), Tag.STAGE.toString(), stage)); } // endregion @@ -130,10 +129,12 @@ public class QueryExecutionMetricSet implements IMetricSet { DRIVER_INTERNAL_PROCESS, WAIT_FOR_RESULT) .forEach( - stage -> { - metricService.remove( - MetricType.TIMER, Metric.QUERY_EXECUTION.toString(), Tag.STAGE.toString(), stage); - }); + stage -> + metricService.remove( + MetricType.TIMER, + Metric.QUERY_EXECUTION.toString(), + Tag.STAGE.toString(), + stage)); } // endregion @@ -160,10 +161,9 @@ public class QueryExecutionMetricSet implements IMetricSet { aggregationFromStatisticsTimer = DoNothingMetricManager.DO_NOTHING_TIMER; Arrays.asList("raw_data", "statistics") .forEach( - from -> { - metricService.remove( - MetricType.TIMER, Metric.AGGREGATION.toString(), Tag.FROM.toString(), from); - }); + from -> + metricService.remove( + MetricType.TIMER, Metric.AGGREGATION.toString(), Tag.FROM.toString(), from)); } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java index 251670bc3e4..da775515688 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryPlanCostMetricSet.java @@ -110,10 +110,12 @@ public class QueryPlanCostMetricSet implements IMetricSet { Arrays.asList( ANALYZER, LOGICAL_PLANNER, DISTRIBUTION_PLANNER, PARTITION_FETCHER, SCHEMA_FETCHER) .forEach( - stage -> { - metricService.remove( - MetricType.TIMER, Metric.QUERY_PLAN_COST.toString(), Tag.STAGE.toString(), stage); - }); + stage -> + metricService.remove( + MetricType.TIMER, + Metric.QUERY_PLAN_COST.toString(), + Tag.STAGE.toString(), + stage)); } public static QueryPlanCostMetricSet getInstance() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java index b8280fe399f..93626b06290 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/QueryRelatedResourceMetricSet.java @@ -27,14 +27,25 @@ import org.apache.iotdb.db.queryengine.execution.memory.MemoryPool; import org.apache.iotdb.db.queryengine.plan.Coordinator; import org.apache.iotdb.db.queryengine.plan.planner.LocalExecutionPlanner; import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Gauge; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; +import static org.apache.iotdb.commons.service.metric.enums.Metric.FRAGMENT_INSTANCE_STATISTICS; + public class QueryRelatedResourceMetricSet implements IMetricSet { + + private static final QueryRelatedResourceMetricSet INSTANCE = new QueryRelatedResourceMetricSet(); + + private QueryRelatedResourceMetricSet() { + // empty constructor + } + // Coordinator private static final Coordinator coordinator = Coordinator.getInstance(); - private static final String COORDINATOR = Metric.COORDINATOR.toString(); + private static final String METRIC_COORDINATOR = Metric.COORDINATOR.toString(); private static final String QUERY_EXECUTION_MAP_SIZE = "query_execution_map_size"; // FragmentInstanceManager @@ -63,11 +74,25 @@ public class QueryRelatedResourceMetricSet implements IMetricSet { private static final String LOCAL_EXECUTION_PLANNER = Metric.LOCAL_EXECUTION_PLANNER.toString(); private static final String FREE_MEMORY_FOR_OPERATORS = "free_memory_for_operators"; + // FragmentInstanceStatistics + public static final String QUERY_FRAGMENT_INSTANCE_COUNT = "query_fragment_instance_count"; + private Gauge fragmentInstanceCountGauge = DoNothingMetricManager.DO_NOTHING_GAUGE; + + public void recordExecutionCount(String stage, long count) { + switch (stage) { + case QUERY_FRAGMENT_INSTANCE_COUNT: + fragmentInstanceCountGauge.set(count); + break; + default: + break; + } + } + @Override public void bindTo(AbstractMetricService metricService) { // Coordinator metricService.createAutoGauge( - COORDINATOR, + METRIC_COORDINATOR, MetricLevel.IMPORTANT, coordinator, Coordinator::getQueryExecutionMapSize, @@ -124,13 +149,21 @@ public class QueryRelatedResourceMetricSet implements IMetricSet { LocalExecutionPlanner::getFreeMemoryForOperators, Tag.NAME.toString(), FREE_MEMORY_FOR_OPERATORS); + + // FragmentInstanceStatistics + fragmentInstanceCountGauge = + metricService.getOrCreateGauge( + FRAGMENT_INSTANCE_STATISTICS.toString(), + MetricLevel.IMPORTANT, + Tag.NAME.toString(), + QUERY_FRAGMENT_INSTANCE_COUNT); } @Override public void unbindFrom(AbstractMetricService metricService) { // Coordinator metricService.remove( - MetricType.AUTO_GAUGE, COORDINATOR, Tag.NAME.toString(), QUERY_EXECUTION_MAP_SIZE); + MetricType.AUTO_GAUGE, METRIC_COORDINATOR, Tag.NAME.toString(), QUERY_EXECUTION_MAP_SIZE); // FragmentInstanceManager metricService.remove( @@ -158,5 +191,16 @@ public class QueryRelatedResourceMetricSet implements IMetricSet { LOCAL_EXECUTION_PLANNER, Tag.NAME.toString(), FREE_MEMORY_FOR_OPERATORS); + + // FragmentInstanceStatistics + metricService.remove( + MetricType.GAUGE, + FRAGMENT_INSTANCE_STATISTICS.toString(), + Tag.NAME.toString(), + QUERY_FRAGMENT_INSTANCE_COUNT); + } + + public static QueryRelatedResourceMetricSet getInstance() { + return INSTANCE; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java index ce33e169b36..8f365ccb886 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.execution.exchange.source.ISourceHandle; import org.apache.iotdb.db.queryengine.execution.exchange.source.SourceHandle; import org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet; +import org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; import org.apache.iotdb.db.queryengine.plan.analyze.Analyzer; import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; @@ -92,6 +93,7 @@ import static com.google.common.base.Throwables.throwIfUnchecked; import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode; import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.WAIT_FOR_RESULT; import static org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.DISTRIBUTION_PLANNER; +import static org.apache.iotdb.db.queryengine.metric.QueryRelatedResourceMetricSet.QUERY_FRAGMENT_INSTANCE_COUNT; /** * QueryExecution stores all the status of a query which is being prepared or running inside the MPP @@ -382,6 +384,10 @@ public class QueryExecution implements IQueryExecution { distributedPlan.getInstances().size(), printFragmentInstances(distributedPlan.getInstances())); } + + QueryRelatedResourceMetricSet.getInstance() + .recordExecutionCount(QUERY_FRAGMENT_INSTANCE_COUNT, distributedPlan.getInstances().size()); + // check timeout after building distribution plan because it could be time-consuming in some // cases. checkTimeOutForQuery(); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 14afbbcb927..872f873fd3c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -96,6 +96,7 @@ public enum Metric { DRIVER_SCHEDULER("driver_scheduler"), COORDINATOR("coordinator"), FRAGMENT_INSTANCE_MANAGER("fragment_instance_manager"), + FRAGMENT_INSTANCE_STATISTICS("fragment_instance_statistics"), MEMORY_POOL("memory_pool"), LOCAL_EXECUTION_PLANNER("local_execution_planner"), // file related
