[doris] branch master updated: [fix](cache) Fix enable sql cache lead to FE Full GC or OOM #22769

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 3b7a0a4713 [fix](cache) Fix enable sql cache lead to FE Full GC or OOM 
#22769
3b7a0a4713 is described below

commit 3b7a0a4713da89fd9a402b43677dda3d7c2998c4
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Wed Aug 9 19:24:04 2023 +0800

    [fix](cache) Fix enable sql cache lead to FE Full GC or OOM #22769
---
 fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java     | 7 ++++++-
 .../src/main/java/org/apache/doris/qe/cache/PartitionCache.java   | 3 +++
 .../src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java  | 8 ++++++++
 fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java  | 3 +++
 4 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
index 31388c99c9..815ca6b0d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/Cache.java
@@ -41,6 +41,7 @@ public abstract class Cache {
     protected TUniqueId queryId;
     protected SelectStmt selectStmt;
     protected RowBatchBuilder rowBatchBuilder;
+    protected boolean disableCache = false;
     protected CacheAnalyzer.CacheTable latestTable;
     protected CacheProxy proxy;
     protected HitRange hitRange;
@@ -81,16 +82,20 @@ public abstract class Cache {
     public abstract void updateCache();
 
     protected boolean checkRowLimit() {
-        if (rowBatchBuilder == null) {
+        if (disableCache || rowBatchBuilder == null) {
             return false;
         }
         if (rowBatchBuilder.getRowSize() > Config.cache_result_max_row_count) {
             LOG.debug("can not be cached. rowbatch size {} is more than {}", 
rowBatchBuilder.getRowSize(),
                     Config.cache_result_max_row_count);
+            rowBatchBuilder.clear();
+            disableCache = true;
             return false;
         } else if (rowBatchBuilder.getDataSize() > 
Config.cache_result_max_data_size) {
             LOG.debug("can not be cached. rowbatch data size {} is more than 
{}", rowBatchBuilder.getDataSize(),
                     Config.cache_result_max_data_size);
+            rowBatchBuilder.clear();
+            disableCache = true;
             return false;
         } else {
             return true;
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
index 9113bfec82..d65d67fdbd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/PartitionCache.java
@@ -123,6 +123,9 @@ public class PartitionCache extends Cache {
             rowBatchBuilder.buildPartitionIndex(selectStmt.getResultExprs(), 
selectStmt.getColLabels(),
                     partColumn, range.buildUpdatePartitionRange());
         }
+        if (!super.checkRowLimit()) {
+            return;
+        }
         rowBatchBuilder.copyRowData(rowBatch);
     }
 
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
index 93f06455f0..2cae79597a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/RowBatchBuilder.java
@@ -103,6 +103,14 @@ public class RowBatchBuilder {
         }
     }
 
+    public void clear() {
+        rowList = Lists.newArrayList();
+        cachePartMap = new HashMap<>();
+        batchSize = 0;
+        rowSize = 0;
+        dataSize = 0;
+    }
+
     public InternalService.PUpdateCacheRequest buildSqlUpdateRequest(
             String sql, long partitionKey, long lastVersion, long lastestTime) 
{
         if (updateRequest == null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
index 550c77ac2a..25b7233770 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/SqlCache.java
@@ -78,6 +78,9 @@ public class SqlCache extends Cache {
         if (rowBatchBuilder == null) {
             rowBatchBuilder = new RowBatchBuilder(CacheAnalyzer.CacheMode.Sql);
         }
+        if (!super.checkRowLimit()) {
+            return;
+        }
         rowBatchBuilder.copyRowData(rowBatch);
     }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org