This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch release-0.10.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 474af05eb50af3943d4eedad9bafdf188d67f273 Author: zhangyue19921010 <[email protected]> AuthorDate: Sat Dec 4 10:56:53 2021 +0800 Revert "[HUDI-2489]Tuning HoodieROTablePathFilter by caching hoodieTableFileSystemView, aiming to reduce unnecessary list/get requests" Co-authored-by: yuezhang <[email protected]> (cherry picked from commit 5616830ae18ab3cbe1592d4e463a940342d25edc) --- .../apache/hudi/hadoop/HoodieROTablePathFilter.java | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java index c797f59..d94018b 100644 --- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java @@ -79,11 +79,6 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial Map<String, HoodieTableMetaClient> metaClientCache; /** - * HoodieTableFileSystemView Cache. - */ - private Map<String, HoodieTableFileSystemView> hoodieTableFileSystemViewCache; - - /** * Hadoop configurations for the FileSystem. */ private SerializableConfiguration conf; @@ -102,7 +97,6 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial this.nonHoodiePathCache = new HashSet<>(); this.conf = new SerializableConfiguration(conf); this.metaClientCache = new HashMap<>(); - this.hoodieTableFileSystemViewCache = new HashMap<>(); } /** @@ -181,15 +175,8 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial metaClientCache.put(baseDir.toString(), metaClient); } - HoodieTableMetaClient finalMetaClient = metaClient; - fsView = hoodieTableFileSystemViewCache.computeIfAbsent(baseDir.toString(), key -> - FileSystemViewManager.createInMemoryFileSystemView( - engineContext, - finalMetaClient, - HoodieInputFormatUtils.buildMetadataConfig(getConf()) - ) - ); - + fsView = FileSystemViewManager.createInMemoryFileSystemView(engineContext, + metaClient, HoodieInputFormatUtils.buildMetadataConfig(getConf())); String partition = FSUtils.getRelativePartitionPath(new Path(metaClient.getBasePath()), folder); List<HoodieBaseFile> latestFiles = fsView.getLatestBaseFiles(partition).collect(Collectors.toList()); // populate the cache @@ -215,6 +202,10 @@ public class HoodieROTablePathFilter implements Configurable, PathFilter, Serial } nonHoodiePathCache.add(folder.toString()); return true; + } finally { + if (fsView != null) { + fsView.close(); + } } } else { // files is at < 3 level depth in FS tree, can't be hoodie dataset
