This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 1ac5ee9a9c7 [MINOR] Cleanup FileSystemViewManager code (#10682)
1ac5ee9a9c7 is described below
commit 1ac5ee9a9c7c99379b600bafb142779886e4529c
Author: voonhous <[email protected]>
AuthorDate: Sun Feb 18 10:33:19 2024 +0800
[MINOR] Cleanup FileSystemViewManager code (#10682)
---
.../common/table/view/FileSystemViewManager.java | 36 ++++++++++------------
.../table/view/HoodieTableFileSystemView.java | 5 ++-
2 files changed, 20 insertions(+), 21 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index d729cc94d10..d5697e83eeb 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -82,7 +82,7 @@ public class FileSystemViewManager {
/**
* Drops reference to File-System Views. Future calls to view results in
creating a new view
*
- * @param basePath
+ * @param basePath Hoodie table base path
*/
public void clearFileSystemView(String basePath) {
SyncableFileSystemView view = globalViewMap.remove(basePath);
@@ -94,7 +94,7 @@ public class FileSystemViewManager {
/**
* Main API to get the file-system view for the base-path.
*
- * @param basePath
+ * @param basePath Hoodie table base path
* @return
*/
public SyncableFileSystemView getFileSystemView(String basePath) {
@@ -130,13 +130,12 @@ public class FileSystemViewManager {
/**
* Create RocksDB based file System view for a table.
*
- * @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
* @param metaClient HoodieTableMetaClient
* @return
*/
- private static RocksDbBasedFileSystemView
createRocksDBBasedFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
+ private static RocksDbBasedFileSystemView
createRocksDBBasedFileSystemView(FileSystemViewStorageConfig viewConf,
+ HoodieTableMetaClient metaClient) {
HoodieTimeline timeline =
metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new RocksDbBasedFileSystemView(metaClient, timeline, viewConf);
}
@@ -144,13 +143,12 @@ public class FileSystemViewManager {
/**
* Create a spillable Map based file System view for a table.
*
- * @param conf Hadoop Configuration
* @param viewConf View Storage Configuration
* @param metaClient HoodieTableMetaClient
* @return
*/
- private static SpillableMapBasedFileSystemView
createSpillableMapBasedFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient,
HoodieCommonConfig commonConfig) {
+ private static SpillableMapBasedFileSystemView
createSpillableMapBasedFileSystemView(FileSystemViewStorageConfig viewConf,
+ HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig) {
LOG.info("Creating SpillableMap based view for basePath " +
metaClient.getBasePath());
HoodieTimeline timeline =
metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf,
commonConfig);
@@ -202,14 +200,13 @@ public class FileSystemViewManager {
/**
* Create a remote file System view for a table.
- *
- * @param conf Hadoop Configuration
+ *
* @param viewConf View Storage Configuration
* @param metaClient Hoodie Table MetaClient for the table.
* @return
*/
- private static RemoteHoodieTableFileSystemView
createRemoteFileSystemView(SerializableConfiguration conf,
- FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient) {
+ private static RemoteHoodieTableFileSystemView
createRemoteFileSystemView(FileSystemViewStorageConfig viewConf,
+ HoodieTableMetaClient metaClient) {
LOG.info("Creating remote view for basePath " + metaClient.getBasePath() +
". Server="
+ viewConf.getRemoteViewServerHost() + ":" +
viewConf.getRemoteViewServerPort() + ", Timeout="
+ viewConf.getRemoteTimelineClientTimeoutSecs());
@@ -241,39 +238,38 @@ public class FileSystemViewManager {
final
HoodieCommonConfig commonConfig,
final
SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata>
metadataCreator) {
LOG.info("Creating View Manager with storage type :" +
config.getStorageType());
- final SerializableConfiguration conf = context.getHadoopConf();
switch (config.getStorageType()) {
case EMBEDDED_KV_STORE:
LOG.info("Creating embedded rocks-db based Table View");
return new FileSystemViewManager(context, config,
- (metaClient, viewConf) -> createRocksDBBasedFileSystemView(conf,
viewConf, metaClient));
+ (metaClient, viewConf) ->
createRocksDBBasedFileSystemView(viewConf, metaClient));
case SPILLABLE_DISK:
LOG.info("Creating Spillable Disk based Table View");
return new FileSystemViewManager(context, config,
- (metaClient, viewConf) ->
createSpillableMapBasedFileSystemView(conf, viewConf, metaClient,
commonConfig));
+ (metaClient, viewConf) ->
createSpillableMapBasedFileSystemView(viewConf, metaClient, commonConfig));
case MEMORY:
LOG.info("Creating in-memory based Table View");
return new FileSystemViewManager(context, config,
(metaClient, viewConfig) ->
createInMemoryFileSystemView(metadataConfig, viewConfig, metaClient,
metadataCreator));
case REMOTE_ONLY:
LOG.info("Creating remote only table view");
- return new FileSystemViewManager(context, config, (metaClient,
viewConfig) -> createRemoteFileSystemView(conf,
- viewConfig, metaClient));
+ return new FileSystemViewManager(context, config, (metaClient,
viewConfig) -> createRemoteFileSystemView(viewConfig,
+ metaClient));
case REMOTE_FIRST:
LOG.info("Creating remote first table view");
return new FileSystemViewManager(context, config, (metaClient,
viewConfig) -> {
RemoteHoodieTableFileSystemView remoteFileSystemView =
- createRemoteFileSystemView(conf, viewConfig, metaClient);
+ createRemoteFileSystemView(viewConfig, metaClient);
SyncableFileSystemView secondaryView;
switch (viewConfig.getSecondaryStorageType()) {
case MEMORY:
secondaryView = createInMemoryFileSystemView(metadataConfig,
viewConfig, metaClient, metadataCreator);
break;
case EMBEDDED_KV_STORE:
- secondaryView = createRocksDBBasedFileSystemView(conf,
viewConfig, metaClient);
+ secondaryView = createRocksDBBasedFileSystemView(viewConfig,
metaClient);
break;
case SPILLABLE_DISK:
- secondaryView = createSpillableMapBasedFileSystemView(conf,
viewConfig, metaClient, commonConfig);
+ secondaryView =
createSpillableMapBasedFileSystemView(viewConfig, metaClient, commonConfig);
break;
default:
throw new IllegalArgumentException("Secondary Storage type can
only be in-memory or spillable. Was :"
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
index f1b56ebe519..427258ff596 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java
@@ -115,6 +115,9 @@ public class HoodieTableFileSystemView extends
IncrementalTimelineSyncFileSystem
super.init(metaClient, visibleActiveTimeline);
}
+ /**
+ * Visible for testing
+ */
public void init(HoodieTableMetaClient metaClient, HoodieTimeline
visibleActiveTimeline,
FileStatus[] fileStatuses) {
init(metaClient, visibleActiveTimeline);
@@ -421,7 +424,7 @@ public class HoodieTableFileSystemView extends
IncrementalTimelineSyncFileSystem
/**
* Get the latest file slices for a given partition including the inflight
ones.
*
- * @param partitionPath
+ * @param partitionPath The partition path of interest
* @return Stream of latest {@link FileSlice} in the partition path.
*/
public Stream<FileSlice> fetchLatestFileSlicesIncludingInflight(String
partitionPath) {
