This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ycy/planCacheDemo
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/ycy/planCacheDemo by this push:
new d7e3b2186ad support basic procedure
d7e3b2186ad is described below
commit d7e3b2186adb8851b8f306ff33a4a9450ddd6693
Author: YangCaiyin <[email protected]>
AuthorDate: Tue Feb 25 13:47:58 2025 +0800
support basic procedure
---
.../db/queryengine/common/MPPQueryContext.java | 14 +++++++++-
.../plan/relational/planner/CachedValue.java | 9 ++++++
.../plan/relational/planner/PlanCacheManager.java | 4 ++-
.../relational/planner/TableLogicalPlanner.java | 32 ++++++++++++++++++----
.../optimizations/PushPredicateIntoTableScan.java | 1 +
.../plan/relational/sql/ast/BinaryLiteral.java | 8 +++++-
.../plan/relational/sql/ast/StringLiteral.java | 11 ++++++--
7 files changed, 68 insertions(+), 11 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
index 38c81c472dd..6b3a37e3996 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java
@@ -28,16 +28,18 @@ import
org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider;
import org.apache.iotdb.db.queryengine.plan.analyze.lock.SchemaLockType;
import
org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager;
import
org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
+import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
import org.apache.iotdb.db.queryengine.statistics.QueryPlanStatistics;
-
import org.apache.tsfile.read.filter.basic.Filter;
import java.time.ZoneId;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
@@ -364,6 +366,12 @@ public class MPPQueryContext {
List<Expression> metaDataExpressionList;
List<String> attributeColumns;
List<Literal> literalList;
+ Map<Symbol, ColumnSchema> assignments;
+
+
+ public void setAssignments(Map<Symbol, ColumnSchema> assignments) {
+ this.assignments = assignments;
+ }
public void setAttributeColumns(List<String> attributeColumns) {
this.attributeColumns = attributeColumns;
@@ -377,6 +385,10 @@ public class MPPQueryContext {
this.literalList = literalList;
}
+ public Map<Symbol, ColumnSchema> getAssignments() {
+ return assignments;
+ }
+
public List<Expression> getMetaDataExpressionList() {
return metaDataExpressionList;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CachedValue.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CachedValue.java
index ceebd607782..4bb68029e97 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CachedValue.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CachedValue.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
@@ -9,6 +10,7 @@ import org.apache.tsfile.read.common.type.Type;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
public class CachedValue {
@@ -16,6 +18,7 @@ public class CachedValue {
DatasetHeader respHeader;
HashMap<Symbol, Type> symbolMap;
+ Map<Symbol, ColumnSchema> assignments;
// Used for indexScan to fetch device
List<Expression> metadataExpressionList;
@@ -27,11 +30,13 @@ public class CachedValue {
List<Literal> literalReference,
DatasetHeader header,
HashMap<Symbol, Type> symbolMap,
+ Map<Symbol, ColumnSchema> assignments,
List<Expression> metadataExpressionList,
List<String> attributeColumns) {
this.planNode = planNode;
this.respHeader = header;
this.symbolMap = symbolMap;
+ this.assignments = assignments;
this.metadataExpressionList = metadataExpressionList;
this.attributeColumns = attributeColumns;
this.literalReference = literalReference;
@@ -57,6 +62,10 @@ public class CachedValue {
return attributeColumns;
}
+ public Map<Symbol, ColumnSchema> getAssignments() {
+ return assignments;
+ }
+
public List<Literal> getLiteralReference() {
return literalReference;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheManager.java
index 25559d6fd20..bb9f3ed0084 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/PlanCacheManager.java
@@ -2,6 +2,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.planner;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal;
@@ -34,12 +35,13 @@ public class PlanCacheManager {
List<Literal> literalReference,
DatasetHeader header,
HashMap<Symbol, Type> symbolMap,
+ Map<Symbol, ColumnSchema> assignments,
List<Expression> expressionList,
List<String> columnAttributes) {
planCache.put(
cachedKey,
new CachedValue(
- planNodeTree, literalReference, header, symbolMap, expressionList,
columnAttributes));
+ planNodeTree, literalReference, header, symbolMap, assignments,
expressionList, columnAttributes));
}
public CachedValue getCachedValue(String cacheKey) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
index 769c489e52d..d40e46bb289 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableLogicalPlanner.java
@@ -28,6 +28,7 @@ import
org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.protocol.client.ConfigNodeClient;
import org.apache.iotdb.db.queryengine.common.MPPQueryContext;
import org.apache.iotdb.db.queryengine.common.QueryId;
import org.apache.iotdb.db.queryengine.common.SessionInfo;
@@ -90,6 +91,8 @@ import org.apache.tsfile.read.common.type.StringType;
import org.apache.tsfile.read.common.type.TypeFactory;
import org.apache.tsfile.read.filter.basic.Filter;
import org.apache.tsfile.utils.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
@@ -102,6 +105,7 @@ import static java.util.Objects.requireNonNull;
import static
org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.LOGICAL_PLANNER;
import static
org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.LOGICAL_PLAN_OPTIMIZE;
import static
org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.PARTITION_FETCHER;
+import static
org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.SCHEMA_FETCHER;
import static
org.apache.iotdb.db.queryengine.metric.QueryPlanCostMetricSet.TABLE_TYPE;
import static
org.apache.iotdb.db.queryengine.plan.analyze.AnalyzeVisitor.getTimePartitionSlotList;
import static
org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice.COUNT_DEVICE_HEADER_STRING;
@@ -161,7 +165,7 @@ public class TableLogicalPlanner {
sb.append(queryContext.getZoneId());
return sb.toString();
}
-
+ private static final Logger logger =
LoggerFactory.getLogger(TableLogicalPlanner.class);
private static final IoTDBConfig CONFIG =
IoTDBDescriptor.getInstance().getConfig();
private DataPartition fetchDataPartitionByDevices(
@@ -202,6 +206,7 @@ public class TableLogicalPlanner {
}
return;
}
+ long startTime = System.nanoTime();
DeviceTableScanNode deviceTableScanNode = (DeviceTableScanNode) planNode;
final List<DeviceEntry> deviceEntries =
metadata.indexScan(
@@ -210,12 +215,17 @@ public class TableLogicalPlanner {
.map(
expression ->
ReplaceSymbolInExpression.transform(
- expression, deviceTableScanNode.getAssignments()))
+ expression, cachedValue.getAssignments()))
.collect(Collectors.toList()),
cachedValue.getAttributeColumns(),
queryContext);
deviceTableScanNode.setDeviceEntries(deviceEntries);
+ final long schemaFetchCost = System.nanoTime() - startTime;
+ QueryPlanCostMetricSet.getInstance()
+ .recordPlanCost(TABLE_TYPE, SCHEMA_FETCHER, schemaFetchCost);
+ queryContext.setFetchSchemaCost(schemaFetchCost);
+
if (deviceEntries.isEmpty()) {
if (analysis.noAggregates() && !analysis.hasJoinNode()) {
// no device entries, queries(except aggregation and join) can be
finished
@@ -231,7 +241,7 @@ public class TableLogicalPlanner {
deviceTableScanNode.setTimeFilter(timeFilter);
- long startTime = System.nanoTime();
+ startTime = System.nanoTime();
final DataPartition dataPartition =
fetchDataPartitionByDevices(
// for tree view, we need to pass actual tree db name to this
method
@@ -263,21 +273,25 @@ public class TableLogicalPlanner {
}
}
+ boolean enableCache = false;
+
public LogicalQueryPlan plan(final Analysis analysis) {
long startTime = System.nanoTime();
+ long totalStartTime = startTime;
Statement statement = analysis.getStatement();
-
// Try to use plan cache
// We should check if statement gis Query in enablePlanCache() method\
String cachedKey = "";
List<Literal> literalReference = null;
- if (statement instanceof Query) {
+ if (enableCache && statement instanceof Query) {
List<Literal> literalList = generalizeStatement((Query) statement);
cachedKey = calculateCacheKey(statement, analysis);
CachedValue cachedValue =
PlanCacheManager.getInstance().getCachedValue(cachedKey);
if (cachedValue != null) {
// deal with the device stuff
+ long curTime = System.nanoTime();
+ logger.info("CachedKey generated cost time: {}", curTime -
totalStartTime);
symbolAllocator.fill(cachedValue.getSymbolMap());
analysis.setRespDatasetHeader(cachedValue.getRespHeader());
adjustBySchema(cachedValue.planNode, cachedValue, analysis);
@@ -286,6 +300,9 @@ public class TableLogicalPlanner {
cachedValue.getLiteralReference().get(i).replace(literalList.get(i));
}
+ logger.info("Logical plan is cached, adjustment cost time: {}",
System.nanoTime() - curTime);
+ logger.info("Logical plan is cached, cost time: {}", System.nanoTime()
- totalStartTime);
+ logger.info("Logical plan is cached, fetch schema cost time: {}",
queryContext.getFetchPartitionCost() + queryContext.getFetchSchemaCost());
return new LogicalQueryPlan(queryContext, cachedValue.getPlanNode());
}
// Following implementation of plan should be based on the
generalizedStatement
@@ -315,6 +332,7 @@ public class TableLogicalPlanner {
warningCollector,
PlanOptimizersStatsCollector.createPlanOptimizersStatsCollector()));
}
+ logger.info("Logical plan is generated, optimization cost time: {}",
System.nanoTime() - startTime);
long logicalOptimizationCost =
System.nanoTime()
- startTime
@@ -331,10 +349,12 @@ public class TableLogicalPlanner {
literalReference,
analysis.getRespDatasetHeader(),
symbolAllocator.cloneSymbolMap(),
+ queryContext.getAssignments(),
queryContext.getMetaDataExpressionList(),
queryContext.getAttributeColumns());
}
-
+ logger.info("Logical plan is generated, fetch schema cost time: {}",
queryContext.getFetchPartitionCost() + queryContext.getFetchSchemaCost());
+ logger.info("Logical plan is generated, cost time: {}", System.nanoTime()
- totalStartTime);
return new LogicalQueryPlan(queryContext, planNode);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
index dec9834eb15..518f486f487 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java
@@ -567,6 +567,7 @@ public class PushPredicateIntoTableScan implements
PlanOptimizer {
.collect(Collectors.toList());
queryContext.setMetaDataExpressionList(metaDataExpression);
queryContext.setAttributeColumns(attributeColumns);
+ queryContext.setAssignments(tableScanNode.getAssignments());
long startTime = System.nanoTime();
final List<DeviceEntry> deviceEntries =
metadata.indexScan(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
index e11cd7a1182..ea6d5819b45 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/BinaryLiteral.java
@@ -40,7 +40,7 @@ public class BinaryLiteral extends Literal {
private static final CharMatcher HEX_DIGIT_MATCHER =
CharMatcher.inRange('A', 'F').or(CharMatcher.inRange('0',
'9')).precomputed();
- private final byte[] value;
+ private byte[] value;
public BinaryLiteral(String value) {
super(null);
@@ -142,4 +142,10 @@ public class BinaryLiteral extends Literal {
public Object getTsValue() {
return new Binary(value);
}
+
+ @Override
+ public void replace(Literal literal) {
+ BinaryLiteral binaryLiteral = (BinaryLiteral) literal;
+ this.value = ((BinaryLiteral) literal).getValue();
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
index 0bf9c7046fe..f660621dfd4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StringLiteral.java
@@ -32,8 +32,8 @@ import static java.util.Objects.requireNonNull;
public class StringLiteral extends Literal {
- private final String value;
- private final int length;
+ private String value;
+ private int length;
public StringLiteral(String value) {
super(null);
@@ -109,4 +109,11 @@ public class StringLiteral extends Literal {
public Object getTsValue() {
return new Binary(value.getBytes(StandardCharsets.UTF_8));
}
+
+ @Override
+ public void replace(Literal literal) {
+ StringLiteral stringLiteral = (StringLiteral) literal;
+ this.value = stringLiteral.getValue();
+ this.length = stringLiteral.length();
+ }
}
