This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2c154c6d971 [fix](jdbc catalog) Fix Memory Leak by Enabling Weak
References in HikariCP (#39582)
2c154c6d971 is described below
commit 2c154c6d9710c16d6057e0d08ba866521a12bbb0
Author: zy-kkk <[email protected]>
AuthorDate: Fri Aug 23 14:08:19 2024 +0800
[fix](jdbc catalog) Fix Memory Leak by Enabling Weak References in HikariCP
(#39582)
This PR addresses a memory leak issue caused by FastList objects in
HikariCP being retained by ThreadLocal variables, which are not easily
garbage collected in long-running JNI threads. To mitigate this, a
system property com.zaxxer.hikari.useWeakReferences is set to true,
ensuring that WeakReference is used for ThreadLocal objects, allowing
the garbage collector to reclaim memory more effectively.
Even though setting this will affect some performance, solving resource
leaks is relatively more important
Performance difference before and after setting
Before setting:
10 concurrency 0.02-0.05
100 concurrency 0.18-0.4
After setting:
10 concurrency 0.02-0.07
100 concurrency 0.18-0.7
---
.../src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java | 3 ++-
.../src/main/java/org/apache/doris/jdbc/JdbcDataSource.java | 6 ++++--
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
index eea57efbf14..3e681fa1519 100644
---
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
+++
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/BaseJdbcExecutor.java
@@ -94,6 +94,7 @@ public abstract class BaseJdbcExecutor implements
JdbcExecutor {
.setConnectionPoolMaxLifeTime(request.connection_pool_max_life_time)
.setConnectionPoolKeepAlive(request.connection_pool_keep_alive);
JdbcDataSource.getDataSource().setCleanupInterval(request.connection_pool_cache_clear_time);
+ System.setProperty("com.zaxxer.hikari.useWeakReferences", "true");
init(config, request.statement);
this.jdbcDriverVersion = getJdbcDriverVersion();
}
@@ -309,7 +310,7 @@ public abstract class BaseJdbcExecutor implements
JdbcExecutor {
ds.setKeepaliveTime(config.getConnectionPoolMaxLifeTime() / 5L); // default 6
min
}
hikariDataSource = ds;
-
JdbcDataSource.getDataSource().putSource(hikariDataSourceKey, ds);
+
JdbcDataSource.getDataSource().putSource(hikariDataSourceKey, hikariDataSource);
LOG.info("JdbcClient set"
+ " ConnectionPoolMinSize = " +
config.getConnectionPoolMinSize()
+ ", ConnectionPoolMaxSize = " +
config.getConnectionPoolMaxSize()
diff --git
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
index 447566596a7..1aea763852f 100644
---
a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
+++
b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
@@ -59,8 +59,10 @@ public class JdbcDataSource {
}
public void setCleanupInterval(long interval) {
- this.cleanupInterval = interval * 1000L;
- restartCleanupTask();
+ if (this.cleanupInterval != interval * 1000L) {
+ this.cleanupInterval = interval * 1000L;
+ restartCleanupTask();
+ }
}
private synchronized void restartCleanupTask() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
