nsivabalan commented on code in PR #11944:
URL: https://github.com/apache/hudi/pull/11944#discussion_r1765996290
##########
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java:
##########
@@ -1405,15 +1417,36 @@ public HoodieMetadataValidationContext(
.withMetadataIndexColumnStats(enableMetadataTable)
.withEnableRecordIndex(enableMetadataTable)
.build();
- this.fileSystemView =
FileSystemViewManager.createInMemoryFileSystemView(engineContext,
- metaClient, metadataConfig);
+ props.put(FileSystemViewStorageConfig.VIEW_TYPE.key(), viewStorageType);
+ FileSystemViewStorageConfig viewConf =
FileSystemViewStorageConfig.newBuilder().fromProperties(props).build();
+
ValidationUtils.checkArgument(viewConf.getStorageType().name().equals(viewStorageType),
"View storage type not reflected");
+ HoodieCommonConfig commonConfig =
HoodieCommonConfig.newBuilder().fromProperties(props).build();
+ this.fileSystemView = getFileSystemView(engineContext,
+ metaClient, metadataConfig, viewConf, commonConfig);
this.tableMetadata = HoodieTableMetadata.create(
engineContext, metaClient.getStorage(), metadataConfig,
metaClient.getBasePath().toString());
if
(metaClient.getCommitsTimeline().filterCompletedInstants().countInstants() > 0)
{
this.allColumnNameList = getAllColumnNames();
}
}
+ private HoodieTableFileSystemView getFileSystemView(HoodieEngineContext
context,
+ HoodieTableMetaClient
metaClient, HoodieMetadataConfig metadataConfig,
+
FileSystemViewStorageConfig viewConf, HoodieCommonConfig commonConfig) {
+ HoodieTimeline timeline =
metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
+ switch (viewConf.getStorageType()) {
+ case SPILLABLE_DISK:
+ LOG.debug("Creating Spillable Disk based Table View");
+ return new SpillableMapBasedFileSystemView(metaClient, timeline,
viewConf, commonConfig);
+ case MEMORY:
+ LOG.debug("Creating in-memory based Table View");
+ return FileSystemViewManager.createInMemoryFileSystemView(context,
+ metaClient, metadataConfig);
Review Comment:
Some of the other possible values are not valid. for eg, REMOTE_FIRST,
REMOTE_ONLY and hence I resorted to just these two for now.
If need be, we can add the support later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
