This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 5402da01072 [fix](stats) Don't save colToPartitions anymore to save
mem (#27880)
5402da01072 is described below
commit 5402da01072eeb7ff8f7f39da79c29afe11cdc60
Author: AKIRA <[email protected]>
AuthorDate: Fri Dec 1 19:54:07 2023 +0800
[fix](stats) Don't save colToPartitions anymore to save mem (#27880)
pick from master #27879
---
.../main/java/org/apache/doris/statistics/AnalysisInfo.java | 2 --
.../main/java/org/apache/doris/statistics/AnalysisJob.java | 12 ++++++++++++
.../java/org/apache/doris/statistics/AnalysisManager.java | 10 ++++++++++
.../java/org/apache/doris/statistics/StatisticConstants.java | 2 ++
4 files changed, 24 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 88d69e1f2ab..8b0ccfbdb80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -97,10 +97,8 @@ public class AnalysisInfo implements Writable {
public final long tblId;
// TODO: Map here is wired, List is enough
- @SerializedName("colToPartitions")
public final Map<String, Set<String>> colToPartitions;
- @SerializedName("partitionNames")
public final Set<String> partitionNames;
@SerializedName("colName")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
index b5dc2cceb9b..f8f6cda2591 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
@@ -179,6 +179,18 @@ public class AnalysisJob {
public void deregisterJob() {
analysisManager.removeJob(jobInfo.jobId);
analysisManager.analysisJobIdToTaskMap.remove(jobInfo.jobId);
+ for (BaseAnalysisTask task : queryingTask) {
+ task.info.colToPartitions.clear();
+ if (task.info.partitionNames != null) {
+ task.info.partitionNames.clear();
+ }
+ }
+ for (BaseAnalysisTask task : queryFinished) {
+ task.info.colToPartitions.clear();
+ if (task.info.partitionNames != null) {
+ task.info.partitionNames.clear();
+ }
+ }
}
protected void syncLoadStats() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index cd633792354..01e48f422f9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -621,6 +621,10 @@ public class AnalysisManager implements Writable {
tableStats.update(jobInfo, tbl);
logCreateTableStats(tableStats);
}
+ jobInfo.colToPartitions.clear();
+ if (jobInfo.partitionNames != null) {
+ jobInfo.partitionNames.clear();
+ }
}
public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
@@ -800,6 +804,9 @@ public class AnalysisManager implements Writable {
while (analysisJobInfoMap.size() >= Config.analyze_record_limit) {
analysisJobInfoMap.remove(analysisJobInfoMap.pollFirstEntry().getKey());
}
+ if (jobInfo.message != null && jobInfo.message.length() >=
StatisticConstants.MSG_LEN_UPPER_BOUND) {
+ jobInfo.message = jobInfo.message.substring(0,
StatisticConstants.MSG_LEN_UPPER_BOUND);
+ }
this.analysisJobInfoMap.put(jobInfo.jobId, jobInfo);
}
@@ -807,6 +814,9 @@ public class AnalysisManager implements Writable {
while (analysisTaskInfoMap.size() >= Config.analyze_record_limit) {
analysisTaskInfoMap.remove(analysisTaskInfoMap.pollFirstEntry().getKey());
}
+ if (taskInfo.message != null && taskInfo.message.length() >=
StatisticConstants.MSG_LEN_UPPER_BOUND) {
+ taskInfo.message = taskInfo.message.substring(0,
StatisticConstants.MSG_LEN_UPPER_BOUND);
+ }
this.analysisTaskInfoMap.put(taskInfo.taskId, taskInfo);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index 3e2b9c8bc28..695edae3864 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -100,6 +100,8 @@ public class StatisticConstants {
public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 70;
+ public static final int MSG_LEN_UPPER_BOUND = 1024;
+
static {
SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER +
FeConstants.INTERNAL_DB_NAME);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
