This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 3305467ac2f Clear useless query resource ASAP
3305467ac2f is described below
commit 3305467ac2fc231b9b5ec9e333e1233b526cd423
Author: Jackie Tien <[email protected]>
AuthorDate: Tue Jan 16 08:50:06 2024 +0800
Clear useless query resource ASAP
---
.../db/queryengine/execution/driver/DataDriver.java | 1 +
.../queryengine/execution/driver/DataDriverContext.java | 16 ++++++++++++++--
.../execution/fragment/FragmentInstanceContext.java | 4 ++++
.../db/queryengine/plan/execution/QueryExecution.java | 3 +++
.../queryengine/plan/planner/LocalExecutionPlanner.java | 7 +++++--
.../queryengine/plan/planner/plan/LogicalQueryPlan.java | 6 +++++-
.../db/queryengine/plan/planner/plan/PlanFragment.java | 5 +++++
.../plan/scheduler/FragmentInstanceDispatcherImpl.java | 3 +++
8 files changed, 40 insertions(+), 5 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java
index 446aff1b93c..ea658fcaf2b 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriver.java
@@ -103,6 +103,7 @@ public class DataDriver extends Driver {
this.init = true;
} finally {
+ ((DataDriverContext) driverContext).clearSourceOperators();
QUERY_EXECUTION_METRICS.recordExecutionCost(
QUERY_RESOURCE_INIT, System.nanoTime() - startTime);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java
index 2a208875ce0..c51c02c0871 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java
@@ -31,8 +31,10 @@ import java.util.List;
public class DataDriverContext extends DriverContext {
- private final List<PartialPath> paths;
- private final List<DataSourceOperator> sourceOperators;
+ // it will be set to null, after being merged into Parent FIContext
+ private List<PartialPath> paths;
+ // it will be set to null, after QueryDataSource being inited
+ private List<DataSourceOperator> sourceOperators;
public DataDriverContext(FragmentInstanceContext fragmentInstanceContext,
int pipelineId) {
super(fragmentInstanceContext, pipelineId);
@@ -58,6 +60,11 @@ public class DataDriverContext extends DriverContext {
return paths;
}
+ public void clearPaths() {
+ // friendly for gc
+ paths = null;
+ }
+
public IDataRegionForQuery getDataRegion() {
return getFragmentInstanceContext().getDataRegion();
}
@@ -70,6 +77,11 @@ public class DataDriverContext extends DriverContext {
return sourceOperators;
}
+ public void clearSourceOperators() {
+ // friendly for gc
+ sourceOperators = null;
+ }
+
@Override
public DriverContext createSubDriverContext(int pipelineId) {
return new DataDriverContext(this, pipelineId);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
index bf39dec609a..fdaa0b6384d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java
@@ -61,6 +61,8 @@ public class FragmentInstanceContext extends QueryContext {
private IDataRegionForQuery dataRegion;
private Filter globalTimeFilter;
+
+ // it will only be used once, after sharedQueryDataSource being inited, it
will be set to null
private List<PartialPath> sourcePaths;
// Shared by all scan operators in this fragment instance to avoid memory
problem
private QueryDataSource sharedQueryDataSource;
@@ -361,6 +363,8 @@ public class FragmentInstanceContext extends QueryContext {
public synchronized QueryDataSource getSharedQueryDataSource() throws
QueryProcessException {
if (sharedQueryDataSource == null) {
initQueryDataSource(sourcePaths);
+ // friendly for gc
+ sourcePaths = null;
}
return sharedQueryDataSource;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
index a2678200780..03c8b64af09 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/QueryExecution.java
@@ -233,6 +233,9 @@ public class QueryExecution implements IQueryExecution {
PERFORMANCE_OVERVIEW_METRICS.recordPlanCost(System.nanoTime() - startTime);
schedule();
+ // friendly for gc
+ logicalPlan.clearUselessMemory();
+
// set partial insert error message
// When some columns in one insert failed, other column will continue
executing insertion.
// The error message should be return to client, therefore we need to set
it after the insertion
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index 48932a9b60c..3aef9be159e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@ -171,8 +171,11 @@ public class LocalExecutionPlanner {
context
.getPipelineDriverFactories()
.forEach(
- pipeline ->
- sourcePaths.addAll(((DataDriverContext)
pipeline.getDriverContext()).getPaths()));
+ pipeline -> {
+ DataDriverContext dataDriverContext = (DataDriverContext)
pipeline.getDriverContext();
+ sourcePaths.addAll(dataDriverContext.getPaths());
+ dataDriverContext.clearPaths();
+ });
return sourcePaths;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/LogicalQueryPlan.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/LogicalQueryPlan.java
index a1c994a037c..3b80c137804 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/LogicalQueryPlan.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/LogicalQueryPlan.java
@@ -28,7 +28,7 @@ import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode;
public class LogicalQueryPlan {
private final MPPQueryContext context;
- private final PlanNode rootNode;
+ private PlanNode rootNode;
public LogicalQueryPlan(MPPQueryContext context, PlanNode rootNode) {
this.context = context;
@@ -42,4 +42,8 @@ public class LogicalQueryPlan {
public MPPQueryContext getContext() {
return context;
}
+
+ public void clearUselessMemory() {
+ rootNode = null;
+ }
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
index f2d5d73affb..e063225b641 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/PlanFragment.java
@@ -196,6 +196,11 @@ public class PlanFragment {
return root;
}
+ public void clearUselessField() {
+ planNodeTree = null;
+ typeProvider = null;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
index 41e370cf0e6..08498bcb141 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java
@@ -131,6 +131,9 @@ public class FragmentInstanceDispatcherImpl implements
IFragInstanceDispatcher {
RpcUtils.getStatus(
TSStatusCode.INTERNAL_SERVER_ERROR, UNEXPECTED_ERRORS +
t.getMessage())));
} finally {
+ // friendly for gc, clear the plan node tree, for some queries select
all devices, it will
+ // release lots of memory
+ instance.getFragment().clearUselessField();
QUERY_EXECUTION_METRIC_SET.recordExecutionCost(
DISPATCH_READ, System.nanoTime() - startTime);
}
