This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7b6db963b53 [fix](hudi) disable fs.impl.cache to avoid FE OOM (#36402)
7b6db963b53 is described below
commit 7b6db963b53f2f13a9c00ed264116a1ade75c916
Author: Mingyu Chen <[email protected]>
AuthorDate: Mon Jun 17 22:21:46 2024 +0800
[fix](hudi) disable fs.impl.cache to avoid FE OOM (#36402)
Sometime when using Hudi Catalog, there will be lots of objects like
`org.apache.hadoop.hdfs.client.impl.DfsClientConf` in FE memory,
which belongs to the Hadoop FileSystem's Cache.
Disable this Cache to avoid FE OOM by setting
`fs.xxx.impl.disable.cache` to true
---
.../doris/datasource/hive/HiveMetaStoreClientHelper.java | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 952454e4a95..2e7693619b8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -66,6 +66,7 @@ import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -826,6 +827,15 @@ public class HiveMetaStoreClientHelper {
public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
String hudiBasePath = table.getRemoteTable().getSd().getLocation();
Configuration conf = getConfiguration(table);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("try setting 'fs.xxx.impl.disable.cache' to true for
hudi's base path: {}", hudiBasePath);
+ }
+ URI hudiBasePathUri = URI.create(hudiBasePath);
+ String scheme = hudiBasePathUri.getScheme();
+ if (!Strings.isNullOrEmpty(scheme)) {
+ // Avoid using Cache in Hadoop FileSystem, which may cause FE OOM.
+ conf.set("fs." + scheme + ".impl.disable.cache", "true");
+ }
return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
() ->
HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
