This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 1910f2925b7 [fix](statistics)Release StmtExecutor in AnalysisTask
object when sql execution finished to release memory #31212 (#31233)
1910f2925b7 is described below
commit 1910f2925b7a5e23b70bb9798f447ab3938fd20d
Author: Jibing-Li <[email protected]>
AuthorDate: Wed Feb 21 20:49:34 2024 +0800
[fix](statistics)Release StmtExecutor in AnalysisTask object when sql
execution finished to release memory #31212 (#31233)
---
.../java/org/apache/doris/statistics/BaseAnalysisTask.java | 10 +++++++++-
.../java/org/apache/doris/statistics/OlapAnalysisTask.java | 14 ++++++++++++--
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 62a1365e321..68767843507 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -22,6 +22,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.datasource.CatalogIf;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.StmtExecutor;
@@ -304,13 +305,20 @@ public abstract class BaseAnalysisTask {
protected void runQuery(String sql) {
long startTime = System.currentTimeMillis();
+ String queryId = "";
try (AutoCloseConnectContext a =
StatisticsUtil.buildConnectContext()) {
stmtExecutor = new StmtExecutor(a.connectContext, sql);
ColStatsData colStatsData = new
ColStatsData(stmtExecutor.executeInternalQuery().get(0));
Env.getCurrentEnv().getStatisticsCache().syncColStats(colStatsData);
+ queryId = DebugUtil.printId(stmtExecutor.getContext().queryId());
job.appendBuf(this, Collections.singletonList(colStatsData));
} finally {
- LOG.debug("End cost time in secs: " + (System.currentTimeMillis()
- startTime) / 1000);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("End cost time in millisec: " +
(System.currentTimeMillis() - startTime)
+ + " Analyze SQL: " + sql + " QueryId: " + queryId);
+ }
+ // Release the reference to stmtExecutor, reduce memory usage.
+ stmtExecutor = null;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 577dac1c6b6..051fe53f5b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -26,6 +26,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
+import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.qe.AutoCloseConnectContext;
import org.apache.doris.qe.StmtExecutor;
import org.apache.doris.statistics.AnalysisInfo.JobType;
@@ -181,14 +182,23 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
tbl.getName(), col.getName());
return null;
}
+ long startTime = System.currentTimeMillis();
Map<String, String> params = new HashMap<>();
params.put("dbName", db.getFullName());
params.put("colName", StatisticsUtil.escapeColumnName(info.colName));
params.put("tblName", tbl.getName());
params.put("index", getIndex());
StringSubstitutor stringSubstitutor = new StringSubstitutor(params);
- stmtExecutor = new StmtExecutor(context.connectContext,
stringSubstitutor.replace(BASIC_STATS_TEMPLATE));
- return stmtExecutor.executeInternalQuery().get(0);
+ String sql = stringSubstitutor.replace(BASIC_STATS_TEMPLATE);
+ stmtExecutor = new StmtExecutor(context.connectContext, sql);
+ ResultRow resultRow = stmtExecutor.executeInternalQuery().get(0);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cost time in millisec: " + (System.currentTimeMillis()
- startTime)
+ + " Min max SQL: " + sql + " QueryId: " +
DebugUtil.printId(stmtExecutor.getContext().queryId()));
+ }
+ // Release the reference to stmtExecutor, reduce memory usage.
+ stmtExecutor = null;
+ return resultRow;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
