This is an automated email from the ASF dual-hosted git repository. hui pushed a commit to branch lmh/addQueryMetrics in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 08d478707b464b43627a889be9f7bf6bc19e9f72 Author: Minghui Liu <[email protected]> AuthorDate: Thu Dec 15 10:43:47 2022 +0800 add metrics: query_plan_cost --- .../apache/iotdb/metrics/config/MetricConfig.java | 4 +- .../iotdb/commons/service/metric/enums/Metric.java | 3 +- .../iotdb/commons/service/metric/enums/Tag.java | 3 +- .../iotdb/db/engine/cache/ChunkCacheMetrics.java | 3 +- .../cache/TimeSeriesMetadataCacheMetrics.java | 2 +- .../iotdb/db/mpp/metric/QueryMetricsManager.java | 49 +++++++++ .../metric/QueryPlanCostMetrics.java} | 58 +++++------ .../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 114 +++++++++------------ .../apache/iotdb/db/mpp/plan/analyze/Analyzer.java | 11 +- .../db/mpp/plan/execution/QueryExecution.java | 8 ++ .../db/mpp/plan/parser/StatementGenerator.java | 88 ++++++++-------- .../iotdb/db/mpp/plan/planner/LogicalPlanner.java | 9 +- .../db/service/metrics/DataNodeMetricsHelper.java | 4 + 13 files changed, 214 insertions(+), 142 deletions(-) diff --git a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java index a2558f8b9b..b17410e12d 100644 --- a/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java +++ b/metrics/interface/src/main/java/org/apache/iotdb/metrics/config/MetricConfig.java @@ -34,10 +34,10 @@ public class MetricConfig { private MetricFrameType metricFrameType = MetricFrameType.MICROMETER; /** The list of reporters provide metrics for external tool */ - private List<ReporterType> metricReporterList = Collections.emptyList(); + private List<ReporterType> metricReporterList = Collections.singletonList(ReporterType.JMX); /** The level of metric service */ - private MetricLevel metricLevel = MetricLevel.CORE; + private MetricLevel metricLevel = MetricLevel.IMPORTANT; /** The period of async collection of some metrics in second */ private Integer asyncCollectPeriodInSecond = 5; diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index fc534ef595..cc9b017328 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -61,7 +61,8 @@ public enum Metric { THRIFT_CONNECTIONS, THRIFT_ACTIVE_THREADS, IOT_CONSENSUS, - STAGE; + STAGE, + QUERY_PLAN_COST; @Override public String toString() { diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java index 1b02b0161c..65280a22a2 100644 --- a/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java +++ b/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Tag.java @@ -23,7 +23,8 @@ public enum Tag { TYPE, NAME, REGION, - STATUS; + STATUS, + STAGE; @Override public String toString() { diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java index 58903a6191..5b092a3930 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java @@ -29,7 +29,8 @@ import org.apache.iotdb.metrics.utils.MetricType; import java.util.Objects; public class ChunkCacheMetrics implements IMetricSet { - private ChunkCache chunkCache; + + private final ChunkCache chunkCache; public ChunkCacheMetrics(ChunkCache chunkCache) { this.chunkCache = chunkCache; diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCacheMetrics.java b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCacheMetrics.java index 84684f624c..d0a5e66008 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCacheMetrics.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/cache/TimeSeriesMetadataCacheMetrics.java @@ -30,7 +30,7 @@ import java.util.Objects; public class TimeSeriesMetadataCacheMetrics implements IMetricSet { - private TimeSeriesMetadataCache timeSeriesMetadataCache; + private final TimeSeriesMetadataCache timeSeriesMetadataCache; public TimeSeriesMetadataCacheMetrics(TimeSeriesMetadataCache timeSeriesMetadataCache) { this.timeSeriesMetadataCache = timeSeriesMetadataCache; diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java new file mode 100644 index 0000000000..8a4e878733 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryMetricsManager.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.metric; + +import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; +import org.apache.iotdb.metrics.type.Timer; +import org.apache.iotdb.metrics.utils.MetricLevel; + +public class QueryMetricsManager { + + private final MetricService metricService = MetricService.getInstance(); + + public void addPlanCost(String stage, long costTimeInNanos) { + Timer timer = + metricService.getOrCreateTimer( + Metric.QUERY_PLAN_COST.toString(), MetricLevel.IMPORTANT, Tag.STAGE.toString(), stage); + timer.updateNanos(costTimeInNanos); + } + + public static QueryMetricsManager getInstance() { + return QueryMetricsManager.QueryMetricsManagerHolder.INSTANCE; + } + + private static class QueryMetricsManagerHolder { + + private static final QueryMetricsManager INSTANCE = new QueryMetricsManager(); + + private QueryMetricsManagerHolder() {} + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryPlanCostMetrics.java similarity index 54% copy from server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java copy to server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryPlanCostMetrics.java index 58903a6191..122ba3ea23 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/metric/QueryPlanCostMetrics.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.engine.cache; +package org.apache.iotdb.db.mpp.metric; import org.apache.iotdb.commons.service.metric.enums.Metric; import org.apache.iotdb.commons.service.metric.enums.Tag; @@ -26,42 +26,42 @@ import org.apache.iotdb.metrics.metricsets.IMetricSet; import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.metrics.utils.MetricType; -import java.util.Objects; +import java.util.Arrays; +import java.util.List; -public class ChunkCacheMetrics implements IMetricSet { - private ChunkCache chunkCache; +public class QueryPlanCostMetrics implements IMetricSet { - public ChunkCacheMetrics(ChunkCache chunkCache) { - this.chunkCache = chunkCache; - } + public static final String SQL_PARSER = "sql_parser"; + public static final String ANALYZER = "analyzer"; + public static final String LOGICAL_PLANNER = "logical_planner"; + public static final String DISTRIBUTION_PLANNER = "distribution_planner"; - @Override - public void bindTo(AbstractMetricService metricService) { - metricService.createAutoGauge( - Metric.CACHE_HIT.toString(), - MetricLevel.IMPORTANT, - chunkCache, - o -> (long) o.getHitRate(), - Tag.NAME.toString(), - "chunk"); - } + public static final String PARTITION_FETCHER = "partition_fetcher"; + public static final String SCHEMA_FETCHER = "schema_fetcher"; - @Override - public void unbindFrom(AbstractMetricService metricService) { - metricService.remove( - MetricType.AUTO_GAUGE, Metric.CACHE_HIT.toString(), Tag.NAME.toString(), "chunk"); - } + private final String metric = Metric.QUERY_PLAN_COST.toString(); + private final String tagKey = Tag.STAGE.toString(); + + private static final List<String> stages = + Arrays.asList( + SQL_PARSER, + ANALYZER, + LOGICAL_PLANNER, + DISTRIBUTION_PLANNER, + PARTITION_FETCHER, + SCHEMA_FETCHER); @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ChunkCacheMetrics that = (ChunkCacheMetrics) o; - return Objects.equals(chunkCache, that.chunkCache); + public void bindTo(AbstractMetricService metricService) { + for (String stage : stages) { + metricService.getOrCreateTimer(metric, MetricLevel.IMPORTANT, tagKey, stage); + } } @Override - public int hashCode() { - return Objects.hash(chunkCache); + public void unbindFrom(AbstractMetricService metricService) { + for (String stage : stages) { + metricService.remove(MetricType.TIMER, metric, tagKey, stage); + } } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java index 81910ef650..8b582fb05a 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.mpp.common.header.DatasetHeader; import org.apache.iotdb.db.mpp.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.mpp.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.mpp.common.schematree.ISchemaTree; +import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.Coordinator; import org.apache.iotdb.db.mpp.plan.execution.ExecutionResult; import org.apache.iotdb.db.mpp.plan.expression.Expression; @@ -153,6 +154,8 @@ import static org.apache.iotdb.commons.conf.IoTDBConstant.LOSS; import static org.apache.iotdb.commons.conf.IoTDBConstant.ONE_LEVEL_PATH_WILDCARD; import static org.apache.iotdb.db.metadata.MetadataConstant.ALL_RESULT_NODES; import static org.apache.iotdb.db.mpp.common.header.ColumnHeaderConstant.DEVICE; +import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics.PARTITION_FETCHER; +import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics.SCHEMA_FETCHER; import static org.apache.iotdb.db.mpp.plan.analyze.SelectIntoUtils.constructTargetDevice; import static org.apache.iotdb.db.mpp.plan.analyze.SelectIntoUtils.constructTargetMeasurement; import static org.apache.iotdb.db.mpp.plan.analyze.SelectIntoUtils.constructTargetPath; @@ -201,13 +204,16 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> // request schema fetch API logger.debug("[StartFetchSchema]"); + long startTime = System.nanoTime(); ISchemaTree schemaTree; if (queryStatement.isGroupByTag()) { schemaTree = schemaFetcher.fetchSchemaWithTags(patternTree); } else { schemaTree = schemaFetcher.fetchSchema(patternTree); } + QueryMetricsManager.getInstance().addPlanCost(SCHEMA_FETCHER, System.nanoTime() - startTime); logger.debug("[EndFetchSchema]"); + // If there is no leaf node in the schema tree, the query should be completed immediately if (schemaTree.isEmpty()) { if (queryStatement.isSelectInto()) { @@ -228,8 +234,23 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> if (analysis.hasValueFilter()) { throw new SemanticException("Only time filters are supported in LAST query"); } + analyzeOrderBy(analysis, queryStatement); - return analyzeLast(analysis, schemaTree.getAllMeasurement(), schemaTree); + + List<MeasurementPath> allSelectedPath = schemaTree.getAllMeasurement(); + analyzeLastSource(analysis, allSelectedPath); + + // set header + analysis.setRespDatasetHeader(DatasetHeaderFactory.getLastQueryHeader()); + + // fetch partition information + Set<String> deviceSet = + allSelectedPath.stream().map(MeasurementPath::getDevice).collect(Collectors.toSet()); + DataPartition dataPartition = + fetchDataPartitionByDevices(deviceSet, schemaTree, analysis.getGlobalTimeFilter()); + analysis.setDataPartitionInfo(dataPartition); + + return analysis; } List<Pair<Expression, String>> outputExpressions; @@ -328,8 +349,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> analysis.setHasValueFilter(hasValueFilter); } - private Analysis analyzeLast( - Analysis analysis, List<MeasurementPath> allSelectedPath, ISchemaTree schemaTree) { + private void analyzeLastSource(Analysis analysis, List<MeasurementPath> allSelectedPath) { Set<Expression> sourceExpressions; List<SortItem> sortItemList = analysis.getMergeOrderParameter().getSortItemList(); if (sortItemList.size() > 0) { @@ -352,47 +372,7 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> .map(TimeSeriesOperand::new) .collect(Collectors.toCollection(LinkedHashSet::new)); } - analysis.setSourceExpressions(sourceExpressions); - - analysis.setRespDatasetHeader(DatasetHeaderFactory.getLastQueryHeader()); - - Set<String> deviceSet = - allSelectedPath.stream().map(MeasurementPath::getDevice).collect(Collectors.toSet()); - - Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res = - getTimePartitionSlotList(analysis.getGlobalTimeFilter()); - - DataPartition dataPartition; - - // there is no satisfied time range - if (res.left.isEmpty() && !res.right.left) { - dataPartition = - new DataPartition( - Collections.emptyMap(), - CONFIG.getSeriesPartitionExecutorClass(), - CONFIG.getSeriesPartitionSlotNum()); - } else { - Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>(); - for (String devicePath : deviceSet) { - DataPartitionQueryParam queryParam = - new DataPartitionQueryParam(devicePath, res.left, res.right.left, res.right.right); - sgNameToQueryParamsMap - .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key -> new ArrayList<>()) - .add(queryParam); - } - - if (res.right.left || res.right.right) { - dataPartition = - partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap); - } else { - dataPartition = partitionFetcher.getDataPartition(sgNameToQueryParamsMap); - } - } - - analysis.setDataPartitionInfo(dataPartition); - - return analysis; } private Map<Integer, List<Pair<Expression, String>>> analyzeSelect( @@ -1168,28 +1148,34 @@ public class AnalyzeVisitor extends StatementVisitor<Analysis, MPPQueryContext> private DataPartition fetchDataPartitionByDevices( Set<String> deviceSet, ISchemaTree schemaTree, Filter globalTimeFilter) { - Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res = - getTimePartitionSlotList(globalTimeFilter); - // there is no satisfied time range - if (res.left.isEmpty() && !res.right.left) { - return new DataPartition( - Collections.emptyMap(), - CONFIG.getSeriesPartitionExecutorClass(), - CONFIG.getSeriesPartitionSlotNum()); - } - Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>(); - for (String devicePath : deviceSet) { - DataPartitionQueryParam queryParam = - new DataPartitionQueryParam(devicePath, res.left, res.right.left, res.right.right); - sgNameToQueryParamsMap - .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key -> new ArrayList<>()) - .add(queryParam); - } + long startTime = System.nanoTime(); + try { + Pair<List<TTimePartitionSlot>, Pair<Boolean, Boolean>> res = + getTimePartitionSlotList(globalTimeFilter); + // there is no satisfied time range + if (res.left.isEmpty() && !res.right.left) { + return new DataPartition( + Collections.emptyMap(), + CONFIG.getSeriesPartitionExecutorClass(), + CONFIG.getSeriesPartitionSlotNum()); + } + Map<String, List<DataPartitionQueryParam>> sgNameToQueryParamsMap = new HashMap<>(); + for (String devicePath : deviceSet) { + DataPartitionQueryParam queryParam = + new DataPartitionQueryParam(devicePath, res.left, res.right.left, res.right.right); + sgNameToQueryParamsMap + .computeIfAbsent(schemaTree.getBelongedDatabase(devicePath), key -> new ArrayList<>()) + .add(queryParam); + } - if (res.right.left || res.right.right) { - return partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap); - } else { - return partitionFetcher.getDataPartition(sgNameToQueryParamsMap); + if (res.right.left || res.right.right) { + return partitionFetcher.getDataPartitionWithUnclosedTimeRange(sgNameToQueryParamsMap); + } else { + return partitionFetcher.getDataPartition(sgNameToQueryParamsMap); + } + } finally { + QueryMetricsManager.getInstance() + .addPlanCost(PARTITION_FETCHER, System.nanoTime() - startTime); } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java index 010b491e55..679a71627f 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/Analyzer.java @@ -21,9 +21,11 @@ package org.apache.iotdb.db.mpp.plan.analyze; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.mpp.common.MPPQueryContext; +import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.statement.Statement; import static org.apache.iotdb.db.mpp.common.QueryId.mockQueryId; +import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics.ANALYZER; /** Analyze the statement and generate Analysis. */ public class Analyzer { @@ -40,7 +42,14 @@ public class Analyzer { } public Analysis analyze(Statement statement) { - return new AnalyzeVisitor(partitionFetcher, schemaFetcher).process(statement, context); + long startTime = System.nanoTime(); + Analysis analysis = + new AnalyzeVisitor(partitionFetcher, schemaFetcher).process(statement, context); + + if (statement.isQuery()) { + QueryMetricsManager.getInstance().addPlanCost(ANALYZER, System.nanoTime() - startTime); + } + return analysis; } public static void validate(Statement statement) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java index 7cfa0a3fa1..c150183527 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/QueryExecution.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.mpp.execution.QueryState; import org.apache.iotdb.db.mpp.execution.QueryStateMachine; import org.apache.iotdb.db.mpp.execution.exchange.ISourceHandle; import org.apache.iotdb.db.mpp.execution.exchange.MPPDataExchangeService; +import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.analyze.Analyzer; import org.apache.iotdb.db.mpp.plan.analyze.IPartitionFetcher; @@ -81,6 +82,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Throwables.throwIfUnchecked; +import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics.DISTRIBUTION_PLANNER; import static org.apache.iotdb.db.mpp.plan.constant.DataNodeEndPoints.isSameNode; /** @@ -300,8 +302,14 @@ public class QueryExecution implements IQueryExecution { // Generate the distributed plan and split it into fragments public void doDistributedPlan() { + long startTime = System.nanoTime(); DistributionPlanner planner = new DistributionPlanner(this.analysis, this.logicalPlan); this.distributedPlan = planner.planFragments(); + + if (rawStatement.isQuery()) { + QueryMetricsManager.getInstance() + .addPlanCost(DISTRIBUTION_PLANNER, System.nanoTime() - startTime); + } if (isQuery() && logger.isDebugEnabled()) { logger.debug( "distribution plan done. Fragment instance count is {}, details is: \n {}", diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java index 151a475d67..604442462e 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/StatementGenerator.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.metadata.template.TemplateQueryType; import org.apache.iotdb.db.metadata.utils.MetaFormatUtils; +import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.expression.binary.GreaterEqualExpression; import org.apache.iotdb.db.mpp.plan.expression.binary.LessThanExpression; import org.apache.iotdb.db.mpp.plan.expression.binary.LogicAndExpression; @@ -101,6 +102,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics.SQL_PARSER; + /** Convert SQL and RPC requests to {@link Statement}. */ public class StatementGenerator { @@ -472,48 +475,53 @@ public class StatementGenerator { } private static Statement invokeParser(String sql, ZoneId zoneId) { - ASTVisitor astVisitor = new ASTVisitor(); - astVisitor.setZoneId(zoneId); - - CharStream charStream1 = CharStreams.fromString(sql); - - SqlLexer lexer1 = new SqlLexer(charStream1); - lexer1.removeErrorListeners(); - lexer1.addErrorListener(SQLParseError.INSTANCE); - - CommonTokenStream tokens1 = new CommonTokenStream(lexer1); - - IoTDBSqlParser parser1 = new IoTDBSqlParser(tokens1); - parser1.getInterpreter().setPredictionMode(PredictionMode.SLL); - parser1.removeErrorListeners(); - parser1.addErrorListener(SQLParseError.INSTANCE); - - ParseTree tree; + long startTime = System.nanoTime(); try { - // STAGE 1: try with simpler/faster SLL(*) - tree = parser1.singleStatement(); - // if we get here, there was no syntax error and SLL(*) was enough; - // there is no need to try full LL(*) - } catch (Exception ex) { - CharStream charStream2 = CharStreams.fromString(sql); - - SqlLexer lexer2 = new SqlLexer(charStream2); - lexer2.removeErrorListeners(); - lexer2.addErrorListener(SQLParseError.INSTANCE); - - CommonTokenStream tokens2 = new CommonTokenStream(lexer2); - - org.apache.iotdb.db.qp.sql.IoTDBSqlParser parser2 = - new org.apache.iotdb.db.qp.sql.IoTDBSqlParser(tokens2); - parser2.getInterpreter().setPredictionMode(PredictionMode.LL); - parser2.removeErrorListeners(); - parser2.addErrorListener(SQLParseError.INSTANCE); - - // STAGE 2: parser with full LL(*) - tree = parser2.singleStatement(); - // if we get here, it's LL not SLL + ASTVisitor astVisitor = new ASTVisitor(); + astVisitor.setZoneId(zoneId); + + CharStream charStream1 = CharStreams.fromString(sql); + + SqlLexer lexer1 = new SqlLexer(charStream1); + lexer1.removeErrorListeners(); + lexer1.addErrorListener(SQLParseError.INSTANCE); + + CommonTokenStream tokens1 = new CommonTokenStream(lexer1); + + IoTDBSqlParser parser1 = new IoTDBSqlParser(tokens1); + parser1.getInterpreter().setPredictionMode(PredictionMode.SLL); + parser1.removeErrorListeners(); + parser1.addErrorListener(SQLParseError.INSTANCE); + + ParseTree tree; + try { + // STAGE 1: try with simpler/faster SLL(*) + tree = parser1.singleStatement(); + // if we get here, there was no syntax error and SLL(*) was enough; + // there is no need to try full LL(*) + } catch (Exception ex) { + CharStream charStream2 = CharStreams.fromString(sql); + + SqlLexer lexer2 = new SqlLexer(charStream2); + lexer2.removeErrorListeners(); + lexer2.addErrorListener(SQLParseError.INSTANCE); + + CommonTokenStream tokens2 = new CommonTokenStream(lexer2); + + org.apache.iotdb.db.qp.sql.IoTDBSqlParser parser2 = + new org.apache.iotdb.db.qp.sql.IoTDBSqlParser(tokens2); + parser2.getInterpreter().setPredictionMode(PredictionMode.LL); + parser2.removeErrorListeners(); + parser2.addErrorListener(SQLParseError.INSTANCE); + + // STAGE 2: parser with full LL(*) + tree = parser2.singleStatement(); + // if we get here, it's LL not SLL + } + return astVisitor.visit(tree); + } finally { + QueryMetricsManager.getInstance().addPlanCost(SQL_PARSER, System.nanoTime() - startTime); } - return astVisitor.visit(tree); } private static void addMeasurementAndValue( diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java index 308887f53a..cc25e475ac 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LogicalPlanner.java @@ -19,14 +19,16 @@ package org.apache.iotdb.db.mpp.plan.planner; import org.apache.iotdb.db.mpp.common.MPPQueryContext; +import org.apache.iotdb.db.mpp.metric.QueryMetricsManager; import org.apache.iotdb.db.mpp.plan.analyze.Analysis; import org.apache.iotdb.db.mpp.plan.optimization.PlanOptimizer; import org.apache.iotdb.db.mpp.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.mpp.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.mpp.plan.statement.crud.QueryStatement; import java.util.List; +import static org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics.LOGICAL_PLANNER; + /** Generate a logical plan for the statement. */ public class LogicalPlanner { @@ -39,10 +41,13 @@ public class LogicalPlanner { } public LogicalQueryPlan plan(Analysis analysis) { + long startTime = System.nanoTime(); PlanNode rootNode = new LogicalPlanVisitor(analysis).process(analysis.getStatement(), context); // optimize the query logical plan - if (analysis.getStatement() instanceof QueryStatement) { + if (analysis.getStatement().isQuery()) { + QueryMetricsManager.getInstance().addPlanCost(LOGICAL_PLANNER, System.nanoTime() - startTime); + for (PlanOptimizer optimizer : optimizers) { rootNode = optimizer.optimize(rootNode, context); } diff --git a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index e36b3b653e..be59c2b288 100644 --- a/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/server/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.service.metrics; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.db.mpp.metric.QueryPlanCostMetrics; import org.apache.iotdb.metrics.metricsets.jvm.JvmMetrics; import org.apache.iotdb.metrics.metricsets.logback.LogbackMetrics; @@ -31,5 +32,8 @@ public class DataNodeMetricsHelper { MetricService.getInstance().addMetricSet(new FileMetrics()); MetricService.getInstance().addMetricSet(new ProcessMetrics()); MetricService.getInstance().addMetricSet(new SystemMetrics(true)); + + // bind query related metrics + MetricService.getInstance().addMetricSet(new QueryPlanCostMetrics()); } }
