This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch rel/1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.2 by this push:
new a3f0454b358 [To rel/1.2] Fix slow creation of view when using batch
creation sql (#10277)
a3f0454b358 is described below
commit a3f0454b3589dc3d91fa7bf951446a810dabde4f
Author: Marcos_Zyk <[email protected]>
AuthorDate: Fri Jun 23 16:41:28 2023 +0800
[To rel/1.2] Fix slow creation of view when using batch creation sql
(#10277)
---
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 22 ++++++++------
.../metedata/write/view/CreateLogicalViewNode.java | 34 +++++++++++++++++++++-
2 files changed, 46 insertions(+), 10 deletions(-)
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
index a6fe6a40bcb..a64c1b173dc 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java
@@ -282,6 +282,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
QueryPlanCostMetricSet.getInstance()
.recordPlanCost(SCHEMA_FETCHER, System.nanoTime() - startTime);
}
+ analysis.setSchemaTree(schemaTree);
// extract global time filter from query filter and determine if there
is a value filter
analyzeGlobalTimeFilter(analysis, queryStatement);
@@ -3242,23 +3243,25 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
/**
* Compute how many paths exist, get the schema tree and the number of
existed paths.
*
- * @param pathList the path you want to check
- * @param context the context of your analyzer
* @return a pair of ISchemaTree, and the number of exist paths.
*/
private Pair<ISchemaTree, Integer> fetchSchemaOfPathsAndCount(
- List<PartialPath> pathList, MPPQueryContext context) {
- PathPatternTree pathPatternTree = new PathPatternTree();
- for (PartialPath path : pathList) {
- pathPatternTree.appendPathPattern(path);
+ List<PartialPath> pathList, Analysis analysis, MPPQueryContext context) {
+ ISchemaTree schemaTree = analysis.getSchemaTree();
+ if (schemaTree == null) {
+ // source is not represented by query, thus has not done fetch schema.
+ PathPatternTree pathPatternTree = new PathPatternTree();
+ for (PartialPath path : pathList) {
+ pathPatternTree.appendPathPattern(path);
+ }
+ schemaTree = this.schemaFetcher.fetchSchema(pathPatternTree, context);
}
- ISchemaTree schemaTree = this.schemaFetcher.fetchSchema(pathPatternTree,
context);
// search each path, make sure they all exist.
int numOfExistPaths = 0;
for (PartialPath path : pathList) {
Pair<List<MeasurementPath>, Integer> pathPair =
schemaTree.searchMeasurementPaths(path);
- numOfExistPaths += pathPair.left.size() > 0 ? 1 : 0;
+ numOfExistPaths += !pathPair.left.isEmpty() ? 1 : 0;
}
return new Pair<>(schemaTree, numOfExistPaths);
}
@@ -3290,6 +3293,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
private Pair<List<Expression>, Analysis> analyzeQueryInLogicalViewStatement(
Analysis analysis, QueryStatement queryStatement, MPPQueryContext
context) {
Analysis queryAnalysis = this.visitQuery(queryStatement, context);
+ analysis.setSchemaTree(queryAnalysis.getSchemaTree());
// get all expression from resultColumns
List<Pair<Expression, String>> outputExpressions =
queryAnalysis.getOutputExpressions();
if (queryAnalysis.isFailed()) {
@@ -3329,7 +3333,7 @@ public class AnalyzeVisitor extends
StatementVisitor<Analysis, MPPQueryContext>
}
}
Pair<ISchemaTree, Integer> schemaOfNeedToCheck =
- fetchSchemaOfPathsAndCount(pathsNeedCheck, context);
+ fetchSchemaOfPathsAndCount(pathsNeedCheck, analysis, context);
if (schemaOfNeedToCheck.right != pathsNeedCheck.size()) {
// some source paths is not exist, and could not fetch schema.
analysis.setFinishQueryAfterAnalyze(true);
diff --git
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java
index 1995b0f2c35..817f407da5a 100644
---
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java
+++
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PathDeserializeUtil;
import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import
org.apache.iotdb.db.metadata.plan.schemaregion.write.ICreateLogicalViewPlan;
import
org.apache.iotdb.db.metadata.view.viewExpression.visitor.GetSourcePathsVisitor;
import org.apache.iotdb.db.mpp.plan.analyze.Analysis;
@@ -225,13 +226,44 @@ public class CreateLogicalViewNode extends WritePlanNode
implements ICreateLogic
// split this node into several nodes according to their regionReplicaSet
List<WritePlanNode> result = new ArrayList<>();
+ int maxSingleRequestSize =
+
IoTDBDescriptor.getInstance().getConfig().getMaxMeasurementNumOfInternalRequest();
for (Map.Entry<TRegionReplicaSet, Map<PartialPath, ViewExpression>> entry :
splitMap.entrySet()) {
// for each entry in splitMap, create a plan node.
- result.add(new CreateLogicalViewNode(getPlanNodeId(), entry.getValue(),
entry.getKey()));
+ if (entry.getValue().size() > maxSingleRequestSize) {
+ for (Map<PartialPath, ViewExpression> splitRequest :
+ splitRequest(entry.getValue(), maxSingleRequestSize)) {
+ result.add(new CreateLogicalViewNode(getPlanNodeId(), splitRequest,
entry.getKey()));
+ }
+ } else {
+ result.add(new CreateLogicalViewNode(getPlanNodeId(),
entry.getValue(), entry.getKey()));
+ }
}
return result;
}
+
+ private List<Map<PartialPath, ViewExpression>> splitRequest(
+ Map<PartialPath, ViewExpression> rawRequest, int maxSize) {
+ int num = 0;
+ List<Map<PartialPath, ViewExpression>> result =
+ new ArrayList<>(rawRequest.size() / maxSize + 1);
+ Map<PartialPath, ViewExpression> map = new HashMap<>();
+ for (Map.Entry<PartialPath, ViewExpression> entry : rawRequest.entrySet())
{
+ if (num == maxSize) {
+ result.add(map);
+ num = 0;
+ map = new HashMap<>();
+ }
+ map.put(entry.getKey(), entry.getValue());
+ num++;
+ }
+ if (num > 0) {
+ result.add(map);
+ }
+ return result;
+ }
+
// endregion
public List<PartialPath> getAllTimeSeriesPathInSource() {
