This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 34228a91cb9 Fix some query metric
34228a91cb9 is described below
commit 34228a91cb93dceb8131f6de67eca1eb44f22e03
Author: Jackie Tien <[email protected]>
AuthorDate: Sun Nov 24 15:05:46 2024 +0800
Fix some query metric
---
.../execution/operator/source/SeriesScanUtil.java | 6 +-
.../metric/SeriesScanCostMetricSet.java | 22 +++----
.../plan/planner/TableOperatorGenerator.java | 75 ++++++++++++++++------
.../plan/relational/sql/parser/SqlParser.java | 8 +++
.../iotdb/db/storageengine/buffer/ChunkCache.java | 8 ++-
5 files changed, 83 insertions(+), 36 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
index 15fe07161a6..c46f1e9ca5b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/SeriesScanUtil.java
@@ -49,6 +49,7 @@ import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.read.reader.IPageReader;
import org.apache.tsfile.read.reader.IPointReader;
import org.apache.tsfile.read.reader.page.AlignedPageReader;
+import org.apache.tsfile.read.reader.page.TablePageReader;
import org.apache.tsfile.read.reader.series.PaginationController;
import org.apache.tsfile.utils.Accountable;
import org.apache.tsfile.utils.RamUsageEstimator;
@@ -1228,7 +1229,10 @@ public class SeriesScanUtil implements Accountable {
this.version = new MergeReaderPriority(fileTimestamp, version, offset,
isSeq);
this.data = data;
this.isSeq = isSeq;
- this.isAligned = data instanceof AlignedPageReader || data instanceof
MemAlignedPageReader;
+ this.isAligned =
+ data instanceof AlignedPageReader
+ || data instanceof MemAlignedPageReader
+ || data instanceof TablePageReader;
this.isMem = data instanceof MemPageReader || data instanceof
MemAlignedPageReader;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
index ab6fb12f596..ebaa7c030d9 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/metric/SeriesScanCostMetricSet.java
@@ -1107,28 +1107,28 @@ public class SeriesScanCostMetricSet implements
IMetricSet {
/////////////////////////////////////////////////////////////////////////////////////////////////
private static final String READ_CHUNK = "read_chunk";
private static final String ALL = "all";
- public static final String READ_CHUNK_ALL = READ_CHUNK + "_" + ALL;
+ public static final String READ_CHUNK_CACHE = READ_CHUNK + "_" + CACHE;
public static final String READ_CHUNK_FILE = READ_CHUNK + "_" + FILE;
- private Timer readChunkAllTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
+ private Timer readChunkCacheTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private Timer readChunkFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
private void bindReadChunk(AbstractMetricService metricService) {
- readChunkAllTimer =
+ readChunkCacheTimer =
metricService.getOrCreateTimer(
Metric.SERIES_SCAN_COST.toString(),
MetricLevel.IMPORTANT,
Tag.STAGE.toString(),
- READ_CHUNK,
+ READ_CHUNK_CACHE,
Tag.TYPE.toString(),
NULL,
Tag.FROM.toString(),
- ALL);
+ CACHE);
readChunkFileTimer =
metricService.getOrCreateTimer(
Metric.SERIES_SCAN_COST.toString(),
MetricLevel.IMPORTANT,
Tag.STAGE.toString(),
- READ_CHUNK,
+ READ_CHUNK_FILE,
Tag.TYPE.toString(),
NULL,
Tag.FROM.toString(),
@@ -1136,16 +1136,16 @@ public class SeriesScanCostMetricSet implements
IMetricSet {
}
private void unbindReadChunk(AbstractMetricService metricService) {
- readChunkAllTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
+ readChunkCacheTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
readChunkFileTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
- Arrays.asList(ALL, FILE)
+ Arrays.asList(CACHE, FILE)
.forEach(
from ->
metricService.remove(
MetricType.TIMER,
Metric.SERIES_SCAN_COST.toString(),
Tag.STAGE.toString(),
- READ_CHUNK,
+ READ_CHUNK + "_" + from,
Tag.TYPE.toString(),
NULL,
Tag.FROM.toString(),
@@ -1571,8 +1571,8 @@ public class SeriesScanCostMetricSet implements
IMetricSet {
case READ_TIMESERIES_METADATA_FILE:
readTimeseriesMetadataFileTimer.updateNanos(cost);
break;
- case READ_CHUNK_ALL:
- readChunkAllTimer.updateNanos(cost);
+ case READ_CHUNK_CACHE:
+ readChunkCacheTimer.updateNanos(cost);
break;
case READ_CHUNK_FILE:
readChunkFileTimer.updateNanos(cost);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
index fe0e8671f4a..adce0a3b607 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java
@@ -383,7 +383,7 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
.addOperatorContext(
context.getNextOperatorId(),
node.getPlanNodeId(),
- TableScanNode.class.getSimpleName());
+ TableScanOperator.class.getSimpleName());
int maxTsBlockLineNum =
TSFileDescriptor.getInstance().getConfig().getMaxTsBlockLineNumber();
if (context.getTypeProvider().getTemplatedInfo() != null) {
@@ -1132,7 +1132,7 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
.addOperatorContext(
context.getNextOperatorId(),
node.getPlanNodeId(),
- StreamSortNode.class.getSimpleName());
+ TableStreamSortOperator.class.getSimpleName());
List<TSDataType> dataTypes = getOutputColumnTypes(node,
context.getTypeProvider());
int sortItemsCount = node.getOrderingScheme().getOrderBy().size();
@@ -1176,11 +1176,6 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
@Override
public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) {
- OperatorContext operatorContext =
- context
- .getDriverContext()
- .addOperatorContext(
- context.getNextOperatorId(), node.getPlanNodeId(),
JoinNode.class.getSimpleName());
List<TSDataType> dataTypes = getOutputColumnTypes(node,
context.getTypeProvider());
Operator leftChild = node.getLeftChild().accept(this, context);
@@ -1221,6 +1216,13 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
}
if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.INNER) {
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ TableInnerJoinOperator.class.getSimpleName());
return new TableInnerJoinOperator(
operatorContext,
leftChild,
@@ -1232,6 +1234,13 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
ASC_TIME_COMPARATOR,
dataTypes);
} else if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.FULL) {
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ TableFullOuterJoinOperator.class.getSimpleName());
return new TableFullOuterJoinOperator(
operatorContext,
leftChild,
@@ -1357,25 +1366,28 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
@Override
public Operator visitAggregation(AggregationNode node,
LocalExecutionPlanContext context) {
- OperatorContext operatorContext =
- context
- .getDriverContext()
- .addOperatorContext(
- context.getNextOperatorId(),
- node.getPlanNodeId(),
- AggregationNode.class.getSimpleName());
+
Operator child = node.getChild().accept(this, context);
if (node.getGroupingKeys().isEmpty()) {
- return planGlobalAggregation(node, child, context.getTypeProvider(),
operatorContext);
+ return planGlobalAggregation(node, child, context.getTypeProvider(),
context);
}
- return planGroupByAggregation(node, child, context.getTypeProvider(),
operatorContext);
+ return planGroupByAggregation(node, child, context.getTypeProvider(),
context);
}
private Operator planGlobalAggregation(
- AggregationNode node, Operator child, TypeProvider typeProvider,
OperatorContext context) {
-
+ AggregationNode node,
+ Operator child,
+ TypeProvider typeProvider,
+ LocalExecutionPlanContext context) {
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ AggregationOperator.class.getSimpleName());
Map<Symbol, AggregationNode.Aggregation> aggregationMap =
node.getAggregations();
ImmutableList.Builder<TableAggregator> aggregatorBuilder = new
ImmutableList.Builder<>();
Map<Symbol, Integer> childLayout =
@@ -1393,7 +1405,7 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
typeProvider,
true,
null)));
- return new AggregationOperator(context, child, aggregatorBuilder.build());
+ return new AggregationOperator(operatorContext, child,
aggregatorBuilder.build());
}
// timeColumnName will only be set for AggTableScan.
@@ -1438,7 +1450,7 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
AggregationNode node,
Operator child,
TypeProvider typeProvider,
- OperatorContext operatorContext) {
+ LocalExecutionPlanContext context) {
Map<Symbol, Integer> childLayout =
makeLayoutFromOutputSymbols(node.getChild().getOutputSymbols());
@@ -1458,6 +1470,13 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
buildAggregator(
childLayout, k, v, node.getStep(), typeProvider,
true, null)));
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ StreamingAggregationOperator.class.getSimpleName());
return new StreamingAggregationOperator(
operatorContext,
child,
@@ -1499,6 +1518,13 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
}
List<Integer> preGroupedChannels = preGroupedChannelsBuilder.build();
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ StreamingHashAggregationOperator.class.getSimpleName());
return new StreamingHashAggregationOperator(
operatorContext,
child,
@@ -1522,6 +1548,13 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
(k, v) ->
aggregatorBuilder.add(
buildGroupByAggregator(childLayout, k, v, node.getStep(),
typeProvider)));
+ OperatorContext operatorContext =
+ context
+ .getDriverContext()
+ .addOperatorContext(
+ context.getNextOperatorId(),
+ node.getPlanNodeId(),
+ HashAggregationOperator.class.getSimpleName());
return new HashAggregationOperator(
operatorContext,
@@ -1739,7 +1772,7 @@ public class TableOperatorGenerator extends
PlanVisitor<Operator, LocalExecution
.addOperatorContext(
context.getNextOperatorId(),
node.getPlanNodeId(),
- AggregationTableScanNode.class.getSimpleName());
+ TableAggregationTableScanOperator.class.getSimpleName());
SeriesScanOptions.Builder scanOptionsBuilder =
node.getTimePredicate().isPresent()
? getSeriesScanOptionsBuilder(context,
node.getTimePredicate().get())
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java
index b20fb829326..1ee59aea651 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/SqlParser.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.queryengine.plan.relational.sql.parser;
+import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node;
@@ -54,6 +55,10 @@ import java.util.function.Function;
import static java.util.Objects.requireNonNull;
public class SqlParser {
+
+ private static final PerformanceOverviewMetrics PERFORMANCE_OVERVIEW_METRICS
=
+ PerformanceOverviewMetrics.getInstance();
+
private static final ANTLRErrorListener LEXER_ERROR_LISTENER =
new BaseErrorListener() {
@Override
@@ -131,6 +136,7 @@ public class SqlParser {
Optional<NodeLocation> location,
Function<RelationalSqlParser, ParserRuleContext> parseFunction,
ZoneId zoneId) {
+ long startTime = System.nanoTime();
try {
RelationalSqlLexer lexer =
new RelationalSqlLexer(new
CaseInsensitiveStream(CharStreams.fromString(sql)));
@@ -190,6 +196,8 @@ public class SqlParser {
return new AstBuilder(location.orElse(null), zoneId).visit(tree);
} catch (StackOverflowError e) {
throw new ParsingException(name + " is too large (stack overflow while
parsing)");
+ } finally {
+ PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() -
startTime);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
index 5914272c7cc..a604409035f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
@@ -47,7 +47,7 @@ import java.util.Objects;
import java.util.function.Function;
import java.util.function.LongConsumer;
-import static
org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_ALL;
+import static
org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_CACHE;
import static
org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet.READ_CHUNK_FILE;
/**
@@ -162,11 +162,13 @@ public class ChunkCache {
} finally {
if (chunkLoader.isCacheMiss()) {
cacheMissAdder.accept(1);
+ SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
+ READ_CHUNK_FILE, System.nanoTime() - startTime);
} else {
cacheHitAdder.accept(1);
+ SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
+ READ_CHUNK_CACHE, System.nanoTime() - startTime);
}
- SERIES_SCAN_COST_METRIC_SET.recordSeriesScanCost(
- READ_CHUNK_ALL, System.nanoTime() - startTime);
}
}
