This is an automated email from the ASF dual-hosted git repository.
kangkaisen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4a5164a Fix 'Filesystem closed' in broker load (#3216)
4a5164a is described below
commit 4a5164ab9d1a4b0ca8e9b8a08f876f22a4a2c500
Author: frwrdt <[email protected]>
AuthorDate: Sat Mar 28 09:14:45 2020 +0800
Fix 'Filesystem closed' in broker load (#3216)
---
.../main/java/org/apache/doris/broker/hdfs/FileSystemManager.java | 7 +++++++
1 file changed, 7 insertions(+)
diff --git
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index 3826f21..e801340 100644
---
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -84,11 +84,16 @@ public class FileSystemManager {
private static final String DEFAULT_DFS_CLIENT_FAILOVER_PROXY_PROVIDER =
"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
private static final String FS_DEFAULTFS_KEY = "fs.defaultFS";
+ // If this property is not set to "true", FileSystem instance will be
returned from cache
+ // which is not thread-safe and may cause 'Filesystem closed' exception
when it is closed by other thread.
+ private static final String FS_HDFS_IMPL_DISABLE_CACHE =
"fs.hdfs.impl.disable.cache";
// arguments for s3a
private static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key";
private static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
private static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint";
+ // This property is used like 'fs.hdfs.impl.disable.cache'
+ private static final String FS_S3A_IMPL_DISABLE_CACHE =
"fs.s3a.impl.disable.cache";
private ScheduledExecutorService handleManagementPool =
Executors.newScheduledThreadPool(2);
@@ -332,6 +337,7 @@ public class FileSystemManager {
}
}
+ conf.set(FS_HDFS_IMPL_DISABLE_CACHE, "true");
FileSystem dfsFileSystem = FileSystem.get(pathUri.getUri(),
conf);
fileSystem.setFileSystem(dfsFileSystem);
}
@@ -383,6 +389,7 @@ public class FileSystemManager {
conf.set(FS_S3A_ACCESS_KEY, accessKey);
conf.set(FS_S3A_SECRET_KEY, secretKey);
conf.set(FS_S3A_ENDPOINT, endpoint);
+ conf.set(FS_S3A_IMPL_DISABLE_CACHE, "true");
FileSystem s3AFileSystem = FileSystem.get(pathUri.getUri(),
conf);
fileSystem.setFileSystem(s3AFileSystem);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
