This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2c40e84 Ensure TableMetaClient and FileSystem instances have
exclusive copy of Configuration
2c40e84 is described below
commit 2c40e8419e209767653e2741e653a71bb429d31d
Author: Balaji Varadarajan <[email protected]>
AuthorDate: Thu Jun 20 08:34:56 2019 -0700
Ensure TableMetaClient and FileSystem instances have exclusive copy of
Configuration
---
.../com/uber/hoodie/client/embedded/EmbeddedTimelineService.java | 2 +-
.../java/com/uber/hoodie/common/SerializableConfiguration.java | 8 ++++++--
.../java/com/uber/hoodie/common/table/HoodieTableMetaClient.java | 2 +-
.../com/uber/hoodie/common/table/view/FileSystemViewManager.java | 2 +-
.../java/com/uber/hoodie/timeline/service/TimelineService.java | 9 +++++++--
5 files changed, 16 insertions(+), 7 deletions(-)
diff --git
a/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java
b/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java
index fbd54c9..4c0539a 100644
---
a/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java
+++
b/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java
@@ -68,7 +68,7 @@ public class EmbeddedTimelineService {
}
public void startServer() throws IOException {
- server = new TimelineService(0, viewManager);
+ server = new TimelineService(0, viewManager, hadoopConf.get());
serverPort = server.startService();
logger.info("Started embedded timeline server at " + hostAddr + ":" +
serverPort);
}
diff --git
a/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java
b/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java
index b66e6de..04732e2 100644
---
a/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java
+++
b/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java
@@ -29,11 +29,15 @@ public class SerializableConfiguration implements
Serializable {
private transient Configuration configuration;
public SerializableConfiguration(Configuration configuration) {
- this.configuration = configuration;
+ this.configuration = new Configuration(configuration);
+ }
+
+ public SerializableConfiguration(SerializableConfiguration configuration) {
+ this.configuration = configuration.get();
}
public Configuration get() {
- return configuration;
+ return new Configuration(configuration);
}
private void writeObject(ObjectOutputStream out) throws IOException {
diff --git
a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
index 7551748..c8dd7d2 100644
---
a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
+++
b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
@@ -85,7 +85,7 @@ public class HoodieTableMetaClient implements Serializable {
throws DatasetNotFoundException {
log.info("Loading HoodieTableMetaClient from " + basePath);
this.basePath = basePath;
- this.hadoopConf = new SerializableConfiguration(new Configuration(conf));
+ this.hadoopConf = new SerializableConfiguration(conf);
Path basePathDir = new Path(this.basePath);
this.metaPath = basePath + File.separator + METAFOLDER_NAME;
Path metaPathDir = new Path(this.metaPath);
diff --git
a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java
b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java
index 9d29302..9086a91 100644
---
a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java
+++
b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java
@@ -65,7 +65,7 @@ public class FileSystemViewManager {
public FileSystemViewManager(SerializableConfiguration conf,
FileSystemViewStorageConfig viewStorageConfig,
Function2<String, FileSystemViewStorageConfig, SyncableFileSystemView>
viewCreator) {
- this.conf = conf;
+ this.conf = new SerializableConfiguration(conf);
this.viewStorageConfig = viewStorageConfig;
this.globalViewMap = new ConcurrentHashMap<>();
this.viewCreator = viewCreator;
diff --git
a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java
b/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java
index ec25019..0707bad 100644
---
a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java
+++
b/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java
@@ -50,13 +50,18 @@ public class TimelineService {
return serverPort;
}
- public TimelineService(int serverPort, FileSystemViewManager
globalFileSystemViewManager) throws IOException {
- this.conf = FSUtils.prepareHadoopConf(new Configuration());
+ public TimelineService(int serverPort, FileSystemViewManager
globalFileSystemViewManager,
+ Configuration conf) throws IOException {
+ this.conf = FSUtils.prepareHadoopConf(conf);
this.fs = FileSystem.get(conf);
this.serverPort = serverPort;
this.fsViewsManager = globalFileSystemViewManager;
}
+ public TimelineService(int serverPort, FileSystemViewManager
globalFileSystemViewManager) throws IOException {
+ this(serverPort, globalFileSystemViewManager, new Configuration());
+ }
+
public TimelineService(Config config) throws IOException {
this(config.serverPort, buildFileSystemViewManager(config,
new SerializableConfiguration(FSUtils.prepareHadoopConf(new
Configuration()))));
