This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d0793fadf3b [Fix](broker)fix change afs and bos properties not working
cause of hadoop filesystem cache. (#39117)
d0793fadf3b is described below
commit d0793fadf3b5b091c1fb788744ab0e1075b84fa1
Author: sparrow <[email protected]>
AuthorDate: Sun Aug 11 23:43:35 2024 +0800
[Fix](broker)fix change afs and bos properties not working cause of hadoop
filesystem cache. (#39117)
Due to hadoop filesystem cache changing afs and bos properties not
working，so add some cache disable config to them.
---
.../src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java | 4 ++++
1 file changed, 4 insertions(+)
diff --git
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index ad1a0bb03cc..44b900c058d 100644
---
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -152,6 +152,7 @@ public class FileSystemManager {
private static final String FS_BOS_SECRET_KEY = "fs.bos.secret.access.key";
private static final String FS_BOS_ENDPOINT = "fs.bos.endpoint";
private static final String FS_BOS_IMPL = "fs.bos.impl";
+ private static final String FS_BOS_IMPL_DISABLE_CACHE =
"fs.bos.impl.disable.cache";
private static final String FS_BOS_MULTIPART_UPLOADS_BLOCK_SIZE =
"fs.bos.multipart.uploads.block.size";
// arguments for afs
@@ -159,6 +160,7 @@ public class FileSystemManager {
private static final String HADOOP_JOB_UGI = "hadoop.job.ugi";
private static final String FS_DEFAULT_NAME = "fs.default.name";
private static final String FS_AFS_IMPL = "fs.afs.impl";
+ private static final String FS_AFS_IMPL_DISABLE_CACHE =
"fs.afs.impl.disable.cache";
private static final String DFS_AGENT_PORT = "dfs.agent.port";
private static final String DFS_CLIENT_AUTH_METHOD =
"dfs.client.auth.method";
private static final String DFS_RPC_TIMEOUT = "dfs.rpc.timeout";
@@ -847,6 +849,7 @@ public class FileSystemManager {
conf.set(FS_BOS_SECRET_KEY, secretKey);
conf.set(FS_BOS_ENDPOINT, endpoint);
conf.set(FS_BOS_IMPL,
"org.apache.hadoop.fs.bos.BaiduBosFileSystem");
+ conf.set(FS_BOS_IMPL_DISABLE_CACHE, "true");
conf.set(FS_BOS_MULTIPART_UPLOADS_BLOCK_SIZE,
multiPartUploadBlockSize);
FileSystem bosFileSystem = FileSystem.get(pathUri.getUri(),
conf);
fileSystem.setFileSystem(bosFileSystem);
@@ -1011,6 +1014,7 @@ public class FileSystemManager {
conf.set(HADOOP_JOB_GROUP_NAME, group);
conf.set(FS_DEFAULT_NAME, host);
conf.set(FS_AFS_IMPL, properties.getOrDefault(FS_AFS_IMPL,
"org.apache.hadoop.fs.LiteFileSystem"));
+ conf.set(FS_AFS_IMPL_DISABLE_CACHE, "true");
conf.set(DFS_CLIENT_AUTH_METHOD,
properties.getOrDefault(DFS_CLIENT_AUTH_METHOD, "3"));
conf.set(DFS_AGENT_PORT,
properties.getOrDefault(DFS_AGENT_PORT, "20001"));
conf.set(DFS_RPC_TIMEOUT,
properties.getOrDefault(DFS_RPC_TIMEOUT, "300000"));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
