This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 734e0cfd98199ea20356acbe5a0b3ad842a3174f Author: stayrascal <[email protected]> AuthorDate: Mon Mar 4 08:40:09 2024 +0800 [MINOR] Clean code of FileSystemViewManager (#10797) Co-authored-by: wuzhiping <[email protected]> --- .../java/org/apache/hudi/table/HoodieTable.java | 15 ++- .../TestRemoteFileSystemViewWithMetadataTable.java | 2 +- .../TestTimelineServerBasedWriteMarkers.java | 5 +- .../hudi/testutils/HoodieClientTestUtils.java | 3 +- .../hudi/common/table/HoodieTableMetaClient.java | 6 +- .../common/table/view/FileSystemViewManager.java | 102 ++++++++++----------- .../hudi/timeline/service/TimelineService.java | 6 +- .../TestRemoteHoodieTableFileSystemView.java | 4 +- 8 files changed, 66 insertions(+), 77 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java index 37e7939ab76..d5244ac427c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -120,13 +120,12 @@ import static org.apache.hudi.metadata.HoodieTableMetadataUtil.metadataPartition * @param <O> Type of outputs */ public abstract class HoodieTable<T, I, K, O> implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(HoodieTable.class); protected final HoodieWriteConfig config; protected final HoodieTableMetaClient metaClient; protected final HoodieIndex<?, ?> index; - private SerializableConfiguration hadoopConfiguration; + private final SerializableConfiguration hadoopConfiguration; protected final TaskContextSupplier taskContextSupplier; private final HoodieTableMetadata metadata; private final HoodieStorageLayout storageLayout; @@ -145,7 +144,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable { .build(); this.metadata = HoodieTableMetadata.create(context, metadataConfig, config.getBasePath()); - this.viewManager = FileSystemViewManager.createViewManager(context, config.getMetadataConfig(), config.getViewStorageConfig(), config.getCommonConfig(), unused -> metadata); + this.viewManager = getViewManager(); this.metaClient = metaClient; this.index = getIndex(config, context); this.storageLayout = getStorageLayout(config); @@ -164,7 +163,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable { private synchronized FileSystemViewManager getViewManager() { if (null == viewManager) { - viewManager = FileSystemViewManager.createViewManager(getContext(), config.getMetadataConfig(), config.getViewStorageConfig(), config.getCommonConfig(), unused -> metadata); + viewManager = FileSystemViewManager.createViewManager(getContext(), config.getViewStorageConfig(), config.getCommonConfig(), unused -> metadata); } return viewManager; } @@ -180,8 +179,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable { * @param records hoodieRecords to upsert * @return HoodieWriteMetadata */ - public abstract HoodieWriteMetadata<O> upsert(HoodieEngineContext context, String instantTime, - I records); + public abstract HoodieWriteMetadata<O> upsert(HoodieEngineContext context, String instantTime, I records); /** * Insert a batch of new records into Hoodie table at the supplied instantTime. @@ -190,8 +188,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable { * @param records hoodieRecords to upsert * @return HoodieWriteMetadata */ - public abstract HoodieWriteMetadata<O> insert(HoodieEngineContext context, String instantTime, - I records); + public abstract HoodieWriteMetadata<O> insert(HoodieEngineContext context, String instantTime, I records); /** * Bulk Insert a batch of new records into Hoodie table at the supplied instantTime. @@ -270,7 +267,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable { * @return HoodieWriteMetadata */ public abstract HoodieWriteMetadata<O> bulkInsertPrepped(HoodieEngineContext context, String instantTime, - I preppedRecords, Option<BulkInsertPartitioner> bulkInsertPartitioner); + I preppedRecords, Option<BulkInsertPartitioner> bulkInsertPartitioner); /** * Replaces all the existing records and inserts the specified new records into Hoodie table at the supplied instantTime, diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java index c4e4776009c..3bd053a4a89 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java @@ -116,7 +116,7 @@ public class TestRemoteFileSystemViewWithMetadataTable extends HoodieSparkClient .serverPort(config.getViewStorageConfig().getRemoteViewServerPort()).build(), FileSystem.get(new Configuration()), FileSystemViewManager.createViewManager( - context, config.getMetadataConfig(), config.getViewStorageConfig(), + context, config.getViewStorageConfig(), config.getCommonConfig(), metaClient -> new HoodieBackedTestDelayedTableMetadata( context, config.getMetadataConfig(), metaClient.getBasePathV2().toString(), true))); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java index b27f40e2add..367229b18da 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/marker/TestTimelineServerBasedWriteMarkers.java @@ -20,7 +20,6 @@ package org.apache.hudi.table.marker; import org.apache.hudi.client.common.HoodieSparkEngineContext; import org.apache.hudi.common.config.HoodieCommonConfig; -import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.table.marker.MarkerType; import org.apache.hudi.common.table.view.FileSystemViewManager; @@ -66,15 +65,13 @@ public class TestTimelineServerBasedWriteMarkers extends TestWriteMarkersBase { FileSystemViewStorageConfig storageConf = FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.SPILLABLE_DISK).build(); - HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build(); HoodieLocalEngineContext localEngineContext = new HoodieLocalEngineContext(metaClient.getHadoopConf()); try { timelineService = new TimelineService(localEngineContext, new Configuration(), TimelineService.Config.builder().serverPort(0).enableMarkerRequests(true).build(), FileSystem.get(new Configuration()), - FileSystemViewManager.createViewManager( - localEngineContext, metadataConfig, storageConf, HoodieCommonConfig.newBuilder().build())); + FileSystemViewManager.createViewManager(localEngineContext, storageConf, HoodieCommonConfig.newBuilder().build())); timelineService.startService(); } catch (Exception ex) { throw new RuntimeException(ex); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java index b59b1ea8d67..2413bf2dffd 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java @@ -281,8 +281,7 @@ public class HoodieClientTestUtils { TimelineService.Config.builder().enableMarkerRequests(true) .serverPort(config.getViewStorageConfig().getRemoteViewServerPort()).build(), FileSystem.get(new Configuration()), - FileSystemViewManager.createViewManager(context, config.getMetadataConfig(), - config.getViewStorageConfig(), config.getCommonConfig())); + FileSystemViewManager.createViewManager(context, config.getViewStorageConfig(), config.getCommonConfig())); timelineService.startService(); LOG.info("Timeline service server port: " + timelineServicePort); return timelineService; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index bdcf19caa96..e7d50805b3f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -513,7 +513,7 @@ public class HoodieTableMetaClient implements Serializable { fs.mkdirs(auxiliaryFolder); } - initializeBootstrapDirsIfNotExists(hadoopConf, basePath, fs); + initializeBootstrapDirsIfNotExists(basePath, fs); HoodieTableConfig.create(fs, metaPathDir, props); // We should not use fs.getConf as this might be different from the original configuration // used to create the fs in unit tests @@ -523,7 +523,7 @@ public class HoodieTableMetaClient implements Serializable { return metaClient; } - public static void initializeBootstrapDirsIfNotExists(Configuration hadoopConf, String basePath, FileSystem fs) throws IOException { + public static void initializeBootstrapDirsIfNotExists(String basePath, FileSystem fs) throws IOException { // Create bootstrap index by partition folder if it does not exist final Path bootstrap_index_folder_by_partition = @@ -684,7 +684,7 @@ public class HoodieTableMetaClient implements Serializable { } public void initializeBootstrapDirsIfNotExists() throws IOException { - initializeBootstrapDirsIfNotExists(getHadoopConf(), basePath.toString(), getFs()); + initializeBootstrapDirsIfNotExists(basePath.toString(), getFs()); } private static HoodieTableMetaClient newMetaClient(Configuration conf, String basePath, boolean loadActiveTimelineOnLoad, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java index d5697e83eeb..172b5e41af7 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java @@ -31,7 +31,6 @@ import org.apache.hudi.common.util.ReflectionUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.metadata.HoodieMetadataFileSystemView; import org.apache.hudi.metadata.HoodieTableMetadata; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -66,17 +65,19 @@ public class FileSystemViewManager { private final SerializableConfiguration conf; // The View Storage config used to store file-system views private final FileSystemViewStorageConfig viewStorageConfig; - // Map from Base-Path to View - private final ConcurrentHashMap<String, SyncableFileSystemView> globalViewMap; // Factory Map to create file-system views private final Function2<HoodieTableMetaClient, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator; + // Map from Base-Path to View + private final ConcurrentHashMap<String, SyncableFileSystemView> globalViewMap; - private FileSystemViewManager(HoodieEngineContext context, FileSystemViewStorageConfig viewStorageConfig, + private FileSystemViewManager( + HoodieEngineContext context, + FileSystemViewStorageConfig viewStorageConfig, Function2<HoodieTableMetaClient, FileSystemViewStorageConfig, SyncableFileSystemView> viewCreator) { this.conf = context.getHadoopConf(); this.viewStorageConfig = viewStorageConfig; - this.globalViewMap = new ConcurrentHashMap<>(); this.viewCreator = viewCreator; + this.globalViewMap = new ConcurrentHashMap<>(); } /** @@ -95,7 +96,7 @@ public class FileSystemViewManager { * Main API to get the file-system view for the base-path. * * @param basePath Hoodie table base path - * @return + * @return {@link SyncableFileSystemView} */ public SyncableFileSystemView getFileSystemView(String basePath) { return globalViewMap.computeIfAbsent(basePath, (path) -> { @@ -108,10 +109,10 @@ public class FileSystemViewManager { * Main API to get the file-system view for the base-path. * * @param metaClient HoodieTableMetaClient - * @return + * @return {@link SyncableFileSystemView} */ public SyncableFileSystemView getFileSystemView(HoodieTableMetaClient metaClient) { - return globalViewMap.computeIfAbsent(metaClient.getBasePath(), + return globalViewMap.computeIfAbsent(metaClient.getBasePathV2().toString(), (path) -> viewCreator.apply(metaClient, viewStorageConfig)); } @@ -130,12 +131,12 @@ public class FileSystemViewManager { /** * Create RocksDB based file System view for a table. * - * @param viewConf View Storage Configuration + * @param viewConf View Storage Configuration * @param metaClient HoodieTableMetaClient - * @return + * @return {@link RocksDbBasedFileSystemView} */ private static RocksDbBasedFileSystemView createRocksDBBasedFileSystemView(FileSystemViewStorageConfig viewConf, - HoodieTableMetaClient metaClient) { + HoodieTableMetaClient metaClient) { HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(); return new RocksDbBasedFileSystemView(metaClient, timeline, viewConf); } @@ -143,24 +144,25 @@ public class FileSystemViewManager { /** * Create a spillable Map based file System view for a table. * - * @param viewConf View Storage Configuration + * @param viewConf View Storage Configuration * @param metaClient HoodieTableMetaClient - * @return + * @return {@link SpillableMapBasedFileSystemView} */ - private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystemView(FileSystemViewStorageConfig viewConf, - HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig) { - LOG.info("Creating SpillableMap based view for basePath " + metaClient.getBasePath()); + private static SpillableMapBasedFileSystemView createSpillableMapBasedFileSystemView( + FileSystemViewStorageConfig viewConf, HoodieTableMetaClient metaClient, HoodieCommonConfig commonConfig) { + LOG.info("Creating SpillableMap based view for basePath {}.", metaClient.getBasePathV2()); HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(); return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf, commonConfig); } /** * Create an in-memory file System view for a table. - * */ - private static HoodieTableFileSystemView createInMemoryFileSystemView(HoodieMetadataConfig metadataConfig, FileSystemViewStorageConfig viewConf, - HoodieTableMetaClient metaClient, SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) { - LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePathV2()); + private static HoodieTableFileSystemView createInMemoryFileSystemView( + FileSystemViewStorageConfig viewConf, + HoodieTableMetaClient metaClient, + SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) { + LOG.info("Creating InMemory based view for basePath {}.", metaClient.getBasePathV2()); HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants(); if (metaClient.getTableConfig().isMetadataTableAvailable()) { ValidationUtils.checkArgument(metadataCreator != null, "Metadata supplier is null. Cannot instantiate metadata file system view"); @@ -168,31 +170,30 @@ public class FileSystemViewManager { } if (metaClient.getMetaserverConfig().isMetaserverEnabled()) { return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS, - new Class<?>[] {HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetaserverConfig.class}, + new Class<?>[]{HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetaserverConfig.class}, metaClient, timeline, metaClient.getMetaserverConfig()); } return new HoodieTableFileSystemView(metaClient, timeline, viewConf.isIncrementalTimelineSyncEnabled()); } - public static HoodieTableFileSystemView createInMemoryFileSystemView(HoodieEngineContext engineContext, HoodieTableMetaClient metaClient, - HoodieMetadataConfig metadataConfig) { - + public static HoodieTableFileSystemView createInMemoryFileSystemView( + HoodieEngineContext engineContext, HoodieTableMetaClient metaClient, HoodieMetadataConfig metadataConfig) { return createInMemoryFileSystemViewWithTimeline(engineContext, metaClient, metadataConfig, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); - } - public static HoodieTableFileSystemView createInMemoryFileSystemViewWithTimeline(HoodieEngineContext engineContext, - HoodieTableMetaClient metaClient, - HoodieMetadataConfig metadataConfig, - HoodieTimeline timeline) { - LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePath()); + public static HoodieTableFileSystemView createInMemoryFileSystemViewWithTimeline( + HoodieEngineContext engineContext, + HoodieTableMetaClient metaClient, + HoodieMetadataConfig metadataConfig, + HoodieTimeline timeline) { + LOG.info("Creating InMemory based view for basePath {}.", metaClient.getBasePathV2()); if (metaClient.getTableConfig().isMetadataTableAvailable()) { return new HoodieMetadataFileSystemView(engineContext, metaClient, timeline, metadataConfig); } if (metaClient.getMetaserverConfig().isMetaserverEnabled()) { return (HoodieTableFileSystemView) ReflectionUtils.loadClass(HOODIE_METASERVER_FILE_SYSTEM_VIEW_CLASS, - new Class<?>[] {HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetadataConfig.class}, + new Class<?>[]{HoodieTableMetaClient.class, HoodieTimeline.class, HoodieMetadataConfig.class}, metaClient, timeline, metaClient.getMetaserverConfig()); } return new HoodieTableFileSystemView(metaClient, timeline); @@ -201,43 +202,40 @@ public class FileSystemViewManager { /** * Create a remote file System view for a table. * - * @param viewConf View Storage Configuration + * @param viewConf View Storage Configuration * @param metaClient Hoodie Table MetaClient for the table. - * @return + * @return {@link RemoteHoodieTableFileSystemView} */ private static RemoteHoodieTableFileSystemView createRemoteFileSystemView(FileSystemViewStorageConfig viewConf, - HoodieTableMetaClient metaClient) { - LOG.info("Creating remote view for basePath " + metaClient.getBasePath() + ". Server=" - + viewConf.getRemoteViewServerHost() + ":" + viewConf.getRemoteViewServerPort() + ", Timeout=" - + viewConf.getRemoteTimelineClientTimeoutSecs()); + HoodieTableMetaClient metaClient) { + LOG.info("Creating remote view for basePath {}. Server={}:{}, Timeout={}", metaClient.getBasePathV2(), + viewConf.getRemoteViewServerHost(), viewConf.getRemoteViewServerPort(), viewConf.getRemoteTimelineClientTimeoutSecs()); return new RemoteHoodieTableFileSystemView(metaClient, viewConf); } + public static FileSystemViewManager createViewManagerWithTableMetadata( + final HoodieEngineContext context, + final HoodieMetadataConfig metadataConfig, + final FileSystemViewStorageConfig config, + final HoodieCommonConfig commonConfig) { + return createViewManager(context, config, commonConfig, + metaClient -> HoodieTableMetadata.create(context, metadataConfig, metaClient.getBasePathV2().toString(), true)); + } + public static FileSystemViewManager createViewManager(final HoodieEngineContext context, - final HoodieMetadataConfig metadataConfig, final FileSystemViewStorageConfig config, final HoodieCommonConfig commonConfig) { - return createViewManager(context, metadataConfig, config, commonConfig, null); - } - - public static FileSystemViewManager createViewManagerWithTableMetadata(final HoodieEngineContext context, - final HoodieMetadataConfig metadataConfig, - final FileSystemViewStorageConfig config, - final HoodieCommonConfig commonConfig) { - return createViewManager(context, metadataConfig, config, commonConfig, - metaClient -> HoodieTableMetadata.create(context, metadataConfig, metaClient.getBasePathV2().toString(), true)); + return createViewManager(context, config, commonConfig, null); } /** * Main Factory method for building file-system views. - * */ public static FileSystemViewManager createViewManager(final HoodieEngineContext context, - final HoodieMetadataConfig metadataConfig, final FileSystemViewStorageConfig config, final HoodieCommonConfig commonConfig, final SerializableFunctionUnchecked<HoodieTableMetaClient, HoodieTableMetadata> metadataCreator) { - LOG.info("Creating View Manager with storage type :" + config.getStorageType()); + LOG.info("Creating View Manager with storage type {}.", config.getStorageType()); switch (config.getStorageType()) { case EMBEDDED_KV_STORE: LOG.info("Creating embedded rocks-db based Table View"); @@ -250,7 +248,7 @@ public class FileSystemViewManager { case MEMORY: LOG.info("Creating in-memory based Table View"); return new FileSystemViewManager(context, config, - (metaClient, viewConfig) -> createInMemoryFileSystemView(metadataConfig, viewConfig, metaClient, metadataCreator)); + (metaClient, viewConfig) -> createInMemoryFileSystemView(viewConfig, metaClient, metadataCreator)); case REMOTE_ONLY: LOG.info("Creating remote only table view"); return new FileSystemViewManager(context, config, (metaClient, viewConfig) -> createRemoteFileSystemView(viewConfig, @@ -263,7 +261,7 @@ public class FileSystemViewManager { SyncableFileSystemView secondaryView; switch (viewConfig.getSecondaryStorageType()) { case MEMORY: - secondaryView = createInMemoryFileSystemView(metadataConfig, viewConfig, metaClient, metadataCreator); + secondaryView = createInMemoryFileSystemView(viewConfig, metaClient, metadataCreator); break; case EMBEDDED_KV_STORE: secondaryView = createRocksDBBasedFileSystemView(viewConfig, metaClient); diff --git a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java index adfc734d1c5..59f30ce21a5 100644 --- a/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java @@ -379,20 +379,20 @@ public class TimelineService { case MEMORY: FileSystemViewStorageConfig.Builder inMemConfBuilder = FileSystemViewStorageConfig.newBuilder(); inMemConfBuilder.withStorageType(FileSystemViewStorageType.MEMORY); - return FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, inMemConfBuilder.build(), commonConfig); + return FileSystemViewManager.createViewManager(localEngineContext, inMemConfBuilder.build(), commonConfig); case SPILLABLE_DISK: { FileSystemViewStorageConfig.Builder spillableConfBuilder = FileSystemViewStorageConfig.newBuilder(); spillableConfBuilder.withStorageType(FileSystemViewStorageType.SPILLABLE_DISK) .withBaseStoreDir(config.baseStorePathForFileGroups) .withMaxMemoryForView(config.maxViewMemPerTableInMB * 1024 * 1024L) .withMemFractionForPendingCompaction(config.memFractionForCompactionPerTable); - return FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, spillableConfBuilder.build(), commonConfig); + return FileSystemViewManager.createViewManager(localEngineContext, spillableConfBuilder.build(), commonConfig); } case EMBEDDED_KV_STORE: { FileSystemViewStorageConfig.Builder rocksDBConfBuilder = FileSystemViewStorageConfig.newBuilder(); rocksDBConfBuilder.withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) .withRocksDBPath(config.rocksDBPath); - return FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, rocksDBConfBuilder.build(), commonConfig); + return FileSystemViewManager.createViewManager(localEngineContext, rocksDBConfBuilder.build(), commonConfig); } default: throw new IllegalArgumentException("Invalid view manager storage type :" + config.viewStorageType); diff --git a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java index c9a103e5264..83469785282 100644 --- a/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java +++ b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/service/functional/TestRemoteHoodieTableFileSystemView.java @@ -19,7 +19,6 @@ package org.apache.hudi.timeline.service.functional; import org.apache.hudi.common.config.HoodieCommonConfig; -import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.model.HoodieFileGroup; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -67,14 +66,13 @@ public class TestRemoteHoodieTableFileSystemView extends TestHoodieTableFileSyst protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) { FileSystemViewStorageConfig sConf = FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.SPILLABLE_DISK).build(); - HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder().build(); HoodieCommonConfig commonConfig = HoodieCommonConfig.newBuilder().build(); HoodieLocalEngineContext localEngineContext = new HoodieLocalEngineContext(metaClient.getHadoopConf()); try { server = new TimelineService(localEngineContext, new Configuration(), TimelineService.Config.builder().serverPort(0).build(), FileSystem.get(new Configuration()), - FileSystemViewManager.createViewManager(localEngineContext, metadataConfig, sConf, commonConfig)); + FileSystemViewManager.createViewManager(localEngineContext, sConf, commonConfig)); server.startService(); } catch (Exception ex) { throw new RuntimeException(ex);
