This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch release-0.15.0 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit f5b8088b4ae63b990a3cd41e9a120a971963f84d Author: Y Ethan Guo <[email protected]> AuthorDate: Sat May 25 20:20:34 2024 -0700 [HUDI-7777] Allow HoodieTableMetaClient to take HoodieStorage instance directly (#11303) --- .../hudi/common/table/HoodieTableMetaClient.java | 65 ++++++++++++++-------- .../table/log/AbstractHoodieLogRecordReader.java | 4 +- .../common/table/view/FileSystemViewManager.java | 3 +- .../io/FileBasedInternalSchemaStorageManager.java | 4 +- .../apache/hudi/metadata/BaseTableMetadata.java | 4 +- .../hudi/metadata/HoodieBackedTableMetadata.java | 5 +- .../common/table/HoodieTableMetaserverClient.java | 6 +- 7 files changed, 57 insertions(+), 34 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 42d8cecffc3..f22e50bd7cd 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -122,18 +122,18 @@ public class HoodieTableMetaClient implements Serializable { * Instantiate HoodieTableMetaClient. * Can only be called if table already exists */ - protected HoodieTableMetaClient(StorageConfiguration<?> conf, String basePath, boolean loadActiveTimelineOnLoad, + protected HoodieTableMetaClient(HoodieStorage storage, String basePath, boolean loadActiveTimelineOnLoad, ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion, String payloadClassName, String recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig) { LOG.info("Loading HoodieTableMetaClient from " + basePath); this.consistencyGuardConfig = consistencyGuardConfig; this.fileSystemRetryConfig = fileSystemRetryConfig; - this.storageConf = conf; + this.storageConf = storage.getConf(); + this.storage = storage; this.basePath = new StoragePath(basePath); this.metaPath = new StoragePath(basePath, METAFOLDER_NAME); - this.storage = getStorage(); - TableNotFoundException.checkTableValidity(storage, this.basePath, metaPath); - this.tableConfig = new HoodieTableConfig(storage, metaPath, payloadClassName, recordMergerStrategy); + TableNotFoundException.checkTableValidity(this.storage, this.basePath, metaPath); + this.tableConfig = new HoodieTableConfig(this.storage, metaPath, payloadClassName, recordMergerStrategy); this.tableType = tableConfig.getTableType(); Option<TimelineLayoutVersion> tableConfigVersion = tableConfig.getTimelineLayoutVersion(); if (layoutVersion.isPresent() && tableConfigVersion.isPresent()) { @@ -162,7 +162,7 @@ public class HoodieTableMetaClient implements Serializable { public static HoodieTableMetaClient reload(HoodieTableMetaClient oldMetaClient) { return HoodieTableMetaClient.builder() - .setConf(oldMetaClient.storageConf.newInstance()) + .setStorage(oldMetaClient.getStorage()) .setBasePath(oldMetaClient.basePath.toString()) .setLoadActiveTimelineOnLoad(oldMetaClient.loadActiveTimelineOnLoad) .setConsistencyGuardConfig(oldMetaClient.consistencyGuardConfig) @@ -297,22 +297,29 @@ public class HoodieTableMetaClient implements Serializable { public HoodieStorage getStorage() { if (storage == null) { - HoodieStorage newStorage = HoodieStorageUtils.getStorage(metaPath, getStorageConf()); - ConsistencyGuard consistencyGuard = consistencyGuardConfig.isConsistencyCheckEnabled() - ? new FailSafeConsistencyGuard(newStorage, consistencyGuardConfig) - : new NoOpConsistencyGuard(); - - storage = getIOFactory(newStorage).getStorage(metaPath, - fileSystemRetryConfig.isFileSystemActionRetryEnable(), - fileSystemRetryConfig.getMaxRetryIntervalMs(), - fileSystemRetryConfig.getMaxRetryNumbers(), - fileSystemRetryConfig.getInitialRetryIntervalMs(), - fileSystemRetryConfig.getRetryExceptions(), - consistencyGuard); + storage = getStorage(metaPath, getStorageConf(), consistencyGuardConfig, fileSystemRetryConfig); } return storage; } + private static HoodieStorage getStorage(StoragePath path, + StorageConfiguration<?> storageConf, + ConsistencyGuardConfig consistencyGuardConfig, + FileSystemRetryConfig fileSystemRetryConfig) { + HoodieStorage newStorage = HoodieStorageUtils.getStorage(path, storageConf); + ConsistencyGuard consistencyGuard = consistencyGuardConfig.isConsistencyCheckEnabled() + ? new FailSafeConsistencyGuard(newStorage, consistencyGuardConfig) + : new NoOpConsistencyGuard(); + + return getIOFactory(newStorage).getStorage(path, + fileSystemRetryConfig.isFileSystemActionRetryEnable(), + fileSystemRetryConfig.getMaxRetryIntervalMs(), + fileSystemRetryConfig.getMaxRetryNumbers(), + fileSystemRetryConfig.getInitialRetryIntervalMs(), + fileSystemRetryConfig.getRetryExceptions(), + consistencyGuard); + } + public void setHoodieStorage(HoodieStorage storage) { this.storage = storage; } @@ -666,16 +673,16 @@ public class HoodieTableMetaClient implements Serializable { initializeBootstrapDirsIfNotExists(basePath.toString(), getStorage()); } - private static HoodieTableMetaClient newMetaClient(StorageConfiguration<?> conf, String basePath, boolean loadActiveTimelineOnLoad, + private static HoodieTableMetaClient newMetaClient(HoodieStorage storage, String basePath, boolean loadActiveTimelineOnLoad, ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion, String payloadClassName, String recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig, HoodieMetaserverConfig metaserverConfig) { return metaserverConfig.isMetaserverEnabled() ? (HoodieTableMetaClient) ReflectionUtils.loadClass("org.apache.hudi.common.table.HoodieTableMetaserverClient", - new Class<?>[] {StorageConfiguration.class, String.class, ConsistencyGuardConfig.class, String.class, + new Class<?>[] {HoodieStorage.class, String.class, ConsistencyGuardConfig.class, String.class, FileSystemRetryConfig.class, Option.class, Option.class, HoodieMetaserverConfig.class}, - conf, basePath, consistencyGuardConfig, recordMergerStrategy, fileSystemRetryConfig, + storage, basePath, consistencyGuardConfig, recordMergerStrategy, fileSystemRetryConfig, Option.ofNullable(metaserverConfig.getDatabaseName()), Option.ofNullable(metaserverConfig.getTableName()), metaserverConfig) - : new HoodieTableMetaClient(conf, basePath, + : new HoodieTableMetaClient(storage, basePath, loadActiveTimelineOnLoad, consistencyGuardConfig, layoutVersion, payloadClassName, recordMergerStrategy, fileSystemRetryConfig); } @@ -689,6 +696,7 @@ public class HoodieTableMetaClient implements Serializable { public static class Builder { private StorageConfiguration<?> conf; + private HoodieStorage storage; private String basePath; private boolean loadActiveTimelineOnLoad = false; private String payloadClassName = null; @@ -703,6 +711,11 @@ public class HoodieTableMetaClient implements Serializable { return this; } + public Builder setStorage(HoodieStorage storage) { + this.storage = storage; + return this; + } + public Builder setBasePath(String basePath) { this.basePath = basePath; return this; @@ -750,9 +763,13 @@ public class HoodieTableMetaClient implements Serializable { } public HoodieTableMetaClient build() { - ValidationUtils.checkArgument(conf != null, "Configuration needs to be set to init HoodieTableMetaClient"); + ValidationUtils.checkArgument(conf != null || storage != null, + "Storage configuration or HoodieStorage needs to be set to init HoodieTableMetaClient"); ValidationUtils.checkArgument(basePath != null, "basePath needs to be set to init HoodieTableMetaClient"); - return newMetaClient(conf, basePath, + if (storage == null) { + storage = getStorage(new StoragePath(basePath), conf, consistencyGuardConfig, fileSystemRetryConfig); + } + return newMetaClient(storage, basePath, loadActiveTimelineOnLoad, consistencyGuardConfig, layoutVersion, payloadClassName, recordMergerStrategy, fileSystemRetryConfig, metaserverConfig); } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java index 66d96e8bfea..058320a32ae 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java @@ -65,7 +65,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.hudi.common.table.log.block.HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_BLOCK; import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.COMPACTED_BLOCK_TIMES; import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.INSTANT_TIME; import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.TARGET_INSTANT_TIME; @@ -160,7 +159,8 @@ public abstract class AbstractHoodieLogRecordReader { this.latestInstantTime = latestInstantTime; this.hoodieTableMetaClient = hoodieTableMetaClientOption.orElseGet( () -> HoodieTableMetaClient.builder() - .setConf(storage.getConf().newInstance()).setBasePath(basePath).build()); + .setStorage(storage) + .setBasePath(basePath).build()); // load class from the payload fully qualified class name HoodieTableConfig tableConfig = this.hoodieTableMetaClient.getTableConfig(); this.payloadClassFQN = tableConfig.getPayloadClass(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java index d34952aa0c8..00af75a2371 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java @@ -101,7 +101,8 @@ public class FileSystemViewManager { */ public SyncableFileSystemView getFileSystemView(String basePath) { return globalViewMap.computeIfAbsent(basePath, (path) -> { - HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(conf.newInstance()).setBasePath(path).build(); + HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder() + .setConf(conf.newInstance()).setBasePath(path).build(); return viewCreator.apply(metaClient, viewStorageConfig); }); } diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java index 5737e2dcec0..9d905a09c77 100644 --- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java @@ -71,7 +71,9 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem // make metaClient build lazy private HoodieTableMetaClient getMetaClient() { if (metaClient == null) { - metaClient = HoodieTableMetaClient.builder().setBasePath(baseSchemaPath.getParent().getParent().toString()).setConf(storage.getConf().newInstance()).build(); + metaClient = HoodieTableMetaClient.builder().setBasePath(baseSchemaPath.getParent().getParent().toString()) + .setStorage(storage) + .build(); } return metaClient; } diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java index c3bd5c636c0..254f421284f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java @@ -28,8 +28,8 @@ import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieRecord; -import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.model.HoodieRecordGlobalLocation; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; @@ -90,7 +90,7 @@ public abstract class BaseTableMetadata extends AbstractHoodieTableMetadata { super(engineContext, storage, dataBasePath); this.dataMetaClient = HoodieTableMetaClient.builder() - .setConf(storage.getConf().newInstance()) + .setStorage(storage) .setBasePath(dataBasePath) .build(); diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java index 185791bbbec..31e44b9e212 100644 --- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java @@ -128,7 +128,10 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata { } } else if (this.metadataMetaClient == null) { try { - this.metadataMetaClient = HoodieTableMetaClient.builder().setConf(getStorageConf().newInstance()).setBasePath(metadataBasePath).build(); + this.metadataMetaClient = HoodieTableMetaClient.builder() + .setStorage(storage) + .setBasePath(metadataBasePath) + .build(); this.metadataFileSystemView = getFileSystemView(metadataMetaClient); this.metadataTableConfig = metadataMetaClient.getTableConfig(); } catch (TableNotFoundException e) { diff --git a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java index 56b2893a2cc..055e76f9e2b 100644 --- a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java +++ b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java @@ -33,7 +33,7 @@ import org.apache.hudi.metaserver.client.HoodieMetaserverClient; import org.apache.hudi.metaserver.client.HoodieMetaserverClientProxy; import org.apache.hudi.metaserver.thrift.NoSuchObjectException; import org.apache.hudi.metaserver.thrift.Table; -import org.apache.hudi.storage.StorageConfiguration; +import org.apache.hudi.storage.HoodieStorage; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; @@ -58,10 +58,10 @@ public class HoodieTableMetaserverClient extends HoodieTableMetaClient { private final Table table; private final transient HoodieMetaserverClient metaserverClient; - public HoodieTableMetaserverClient(StorageConfiguration<?> conf, String basePath, ConsistencyGuardConfig consistencyGuardConfig, + public HoodieTableMetaserverClient(HoodieStorage storage, String basePath, ConsistencyGuardConfig consistencyGuardConfig, String mergerStrategy, FileSystemRetryConfig fileSystemRetryConfig, Option<String> databaseName, Option<String> tableName, HoodieMetaserverConfig config) { - super(conf, basePath, false, consistencyGuardConfig, Option.of(TimelineLayoutVersion.CURR_LAYOUT_VERSION), + super(storage, basePath, false, consistencyGuardConfig, Option.of(TimelineLayoutVersion.CURR_LAYOUT_VERSION), config.getString(HoodieTableConfig.PAYLOAD_CLASS_NAME), mergerStrategy, fileSystemRetryConfig); this.databaseName = databaseName.isPresent() ? databaseName.get() : tableConfig.getDatabaseName(); this.tableName = tableName.isPresent() ? tableName.get() : tableConfig.getTableName();
