This is an automated email from the ASF dual-hosted git repository. vbalaji pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push: new 2c40e84 Ensure TableMetaClient and FileSystem instances have exclusive copy of Configuration 2c40e84 is described below commit 2c40e8419e209767653e2741e653a71bb429d31d Author: Balaji Varadarajan <varad...@uber.com> AuthorDate: Thu Jun 20 08:34:56 2019 -0700 Ensure TableMetaClient and FileSystem instances have exclusive copy of Configuration --- .../com/uber/hoodie/client/embedded/EmbeddedTimelineService.java | 2 +- .../java/com/uber/hoodie/common/SerializableConfiguration.java | 8 ++++++-- .../java/com/uber/hoodie/common/table/HoodieTableMetaClient.java | 2 +- .../com/uber/hoodie/common/table/view/FileSystemViewManager.java | 2 +- .../java/com/uber/hoodie/timeline/service/TimelineService.java | 9 +++++++-- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java b/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java index fbd54c9..4c0539a 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java @@ -68,7 +68,7 @@ public class EmbeddedTimelineService { } public void startServer() throws IOException { - server = new TimelineService(0, viewManager); + server = new TimelineService(0, viewManager, hadoopConf.get()); serverPort = server.startService(); logger.info("Started embedded timeline server at " + hostAddr + ":" + serverPort); } diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java b/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java index b66e6de..04732e2 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java @@ -29,11 +29,15 @@ public class SerializableConfiguration implements Serializable { private transient Configuration configuration; public SerializableConfiguration(Configuration configuration) { - this.configuration = configuration; + this.configuration = new Configuration(configuration); + } + + public SerializableConfiguration(SerializableConfiguration configuration) { + this.configuration = configuration.get(); } public Configuration get() { - return configuration; + return new Configuration(configuration); } private void writeObject(ObjectOutputStream out) throws IOException { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java index 7551748..c8dd7d2 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java @@ -85,7 +85,7 @@ public class HoodieTableMetaClient implements Serializable { throws DatasetNotFoundException { log.info("Loading HoodieTableMetaClient from " + basePath); this.basePath = basePath; - this.hadoopConf = new SerializableConfiguration(new Configuration(conf)); + this.hadoopConf = new SerializableConfiguration(conf); Path basePathDir = new Path(this.basePath); this.metaPath = basePath + File.separator + METAFOLDER_NAME; Path metaPathDir = new Path(this.metaPath); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java index 9d29302..9086a91 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java @@ -65,7 +65,7 @@ public class FileSystemViewManager { public FileSystemViewManager(SerializableConfiguration conf, FileSystemViewStorageConfig viewStorageConfig, Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) { - this.conf = conf; + this.conf = new SerializableConfiguration(conf); this.viewStorageConfig = viewStorageConfig; this.globalViewMap = new ConcurrentHashMap<>(); this.viewCreator = viewCreator; diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java b/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java index ec25019..0707bad 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java +++ b/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java @@ -50,13 +50,18 @@ public class TimelineService { return serverPort; } - public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager) throws IOException { - this.conf = FSUtils.prepareHadoopConf(new Configuration()); + public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager, + Configuration conf) throws IOException { + this.conf = FSUtils.prepareHadoopConf(conf); this.fs = FileSystem.get(conf); this.serverPort = serverPort; this.fsViewsManager = globalFileSystemViewManager; } + public TimelineService(int serverPort, FileSystemViewManager globalFileSystemViewManager) throws IOException { + this(serverPort, globalFileSystemViewManager, new Configuration()); + } + public TimelineService(Config config) throws IOException { this(config.serverPort, buildFileSystemViewManager(config, new SerializableConfiguration(FSUtils.prepareHadoopConf(new Configuration()))));