This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/branch-0.x by this push:
new 46b9c3ef87a [HUDI-7777] Allow HoodieTableMetaClient to take
HoodieStorage instance directly (#11303)
46b9c3ef87a is described below
commit 46b9c3ef87a91e0a778a398b16124b81742922d6
Author: Y Ethan Guo <[email protected]>
AuthorDate: Sat May 25 20:20:34 2024 -0700
[HUDI-7777] Allow HoodieTableMetaClient to take HoodieStorage instance
directly (#11303)
---
.../hudi/common/table/HoodieTableMetaClient.java | 65 ++++++++++++++--------
.../table/log/AbstractHoodieLogRecordReader.java | 4 +-
.../common/table/view/FileSystemViewManager.java | 3 +-
.../io/FileBasedInternalSchemaStorageManager.java | 4 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 4 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 5 +-
.../common/table/HoodieTableMetaserverClient.java | 6 +-
7 files changed, 57 insertions(+), 34 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index 42d8cecffc3..f22e50bd7cd 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -122,18 +122,18 @@ public class HoodieTableMetaClient implements
Serializable {
* Instantiate HoodieTableMetaClient.
* Can only be called if table already exists
*/
- protected HoodieTableMetaClient(StorageConfiguration<?> conf, String
basePath, boolean loadActiveTimelineOnLoad,
+ protected HoodieTableMetaClient(HoodieStorage storage, String basePath,
boolean loadActiveTimelineOnLoad,
ConsistencyGuardConfig
consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
String payloadClassName, String
recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig) {
LOG.info("Loading HoodieTableMetaClient from " + basePath);
this.consistencyGuardConfig = consistencyGuardConfig;
this.fileSystemRetryConfig = fileSystemRetryConfig;
- this.storageConf = conf;
+ this.storageConf = storage.getConf();
+ this.storage = storage;
this.basePath = new StoragePath(basePath);
this.metaPath = new StoragePath(basePath, METAFOLDER_NAME);
- this.storage = getStorage();
- TableNotFoundException.checkTableValidity(storage, this.basePath,
metaPath);
- this.tableConfig = new HoodieTableConfig(storage, metaPath,
payloadClassName, recordMergerStrategy);
+ TableNotFoundException.checkTableValidity(this.storage, this.basePath,
metaPath);
+ this.tableConfig = new HoodieTableConfig(this.storage, metaPath,
payloadClassName, recordMergerStrategy);
this.tableType = tableConfig.getTableType();
Option<TimelineLayoutVersion> tableConfigVersion =
tableConfig.getTimelineLayoutVersion();
if (layoutVersion.isPresent() && tableConfigVersion.isPresent()) {
@@ -162,7 +162,7 @@ public class HoodieTableMetaClient implements Serializable {
public static HoodieTableMetaClient reload(HoodieTableMetaClient
oldMetaClient) {
return HoodieTableMetaClient.builder()
- .setConf(oldMetaClient.storageConf.newInstance())
+ .setStorage(oldMetaClient.getStorage())
.setBasePath(oldMetaClient.basePath.toString())
.setLoadActiveTimelineOnLoad(oldMetaClient.loadActiveTimelineOnLoad)
.setConsistencyGuardConfig(oldMetaClient.consistencyGuardConfig)
@@ -297,22 +297,29 @@ public class HoodieTableMetaClient implements
Serializable {
public HoodieStorage getStorage() {
if (storage == null) {
- HoodieStorage newStorage = HoodieStorageUtils.getStorage(metaPath,
getStorageConf());
- ConsistencyGuard consistencyGuard =
consistencyGuardConfig.isConsistencyCheckEnabled()
- ? new FailSafeConsistencyGuard(newStorage, consistencyGuardConfig)
- : new NoOpConsistencyGuard();
-
- storage = getIOFactory(newStorage).getStorage(metaPath,
- fileSystemRetryConfig.isFileSystemActionRetryEnable(),
- fileSystemRetryConfig.getMaxRetryIntervalMs(),
- fileSystemRetryConfig.getMaxRetryNumbers(),
- fileSystemRetryConfig.getInitialRetryIntervalMs(),
- fileSystemRetryConfig.getRetryExceptions(),
- consistencyGuard);
+ storage = getStorage(metaPath, getStorageConf(), consistencyGuardConfig,
fileSystemRetryConfig);
}
return storage;
}
+ private static HoodieStorage getStorage(StoragePath path,
+ StorageConfiguration<?> storageConf,
+ ConsistencyGuardConfig
consistencyGuardConfig,
+ FileSystemRetryConfig
fileSystemRetryConfig) {
+ HoodieStorage newStorage = HoodieStorageUtils.getStorage(path,
storageConf);
+ ConsistencyGuard consistencyGuard =
consistencyGuardConfig.isConsistencyCheckEnabled()
+ ? new FailSafeConsistencyGuard(newStorage, consistencyGuardConfig)
+ : new NoOpConsistencyGuard();
+
+ return getIOFactory(newStorage).getStorage(path,
+ fileSystemRetryConfig.isFileSystemActionRetryEnable(),
+ fileSystemRetryConfig.getMaxRetryIntervalMs(),
+ fileSystemRetryConfig.getMaxRetryNumbers(),
+ fileSystemRetryConfig.getInitialRetryIntervalMs(),
+ fileSystemRetryConfig.getRetryExceptions(),
+ consistencyGuard);
+ }
+
public void setHoodieStorage(HoodieStorage storage) {
this.storage = storage;
}
@@ -666,16 +673,16 @@ public class HoodieTableMetaClient implements
Serializable {
initializeBootstrapDirsIfNotExists(basePath.toString(), getStorage());
}
- private static HoodieTableMetaClient newMetaClient(StorageConfiguration<?>
conf, String basePath, boolean loadActiveTimelineOnLoad,
+ private static HoodieTableMetaClient newMetaClient(HoodieStorage storage,
String basePath, boolean loadActiveTimelineOnLoad,
ConsistencyGuardConfig
consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
String payloadClassName,
String recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig,
HoodieMetaserverConfig metaserverConfig) {
return metaserverConfig.isMetaserverEnabled()
? (HoodieTableMetaClient)
ReflectionUtils.loadClass("org.apache.hudi.common.table.HoodieTableMetaserverClient",
- new Class<?>[] {StorageConfiguration.class, String.class,
ConsistencyGuardConfig.class, String.class,
+ new Class<?>[] {HoodieStorage.class, String.class,
ConsistencyGuardConfig.class, String.class,
FileSystemRetryConfig.class, Option.class, Option.class,
HoodieMetaserverConfig.class},
- conf, basePath, consistencyGuardConfig, recordMergerStrategy,
fileSystemRetryConfig,
+ storage, basePath, consistencyGuardConfig, recordMergerStrategy,
fileSystemRetryConfig,
Option.ofNullable(metaserverConfig.getDatabaseName()),
Option.ofNullable(metaserverConfig.getTableName()), metaserverConfig)
- : new HoodieTableMetaClient(conf, basePath,
+ : new HoodieTableMetaClient(storage, basePath,
loadActiveTimelineOnLoad, consistencyGuardConfig, layoutVersion,
payloadClassName, recordMergerStrategy, fileSystemRetryConfig);
}
@@ -689,6 +696,7 @@ public class HoodieTableMetaClient implements Serializable {
public static class Builder {
private StorageConfiguration<?> conf;
+ private HoodieStorage storage;
private String basePath;
private boolean loadActiveTimelineOnLoad = false;
private String payloadClassName = null;
@@ -703,6 +711,11 @@ public class HoodieTableMetaClient implements Serializable
{
return this;
}
+ public Builder setStorage(HoodieStorage storage) {
+ this.storage = storage;
+ return this;
+ }
+
public Builder setBasePath(String basePath) {
this.basePath = basePath;
return this;
@@ -750,9 +763,13 @@ public class HoodieTableMetaClient implements Serializable
{
}
public HoodieTableMetaClient build() {
- ValidationUtils.checkArgument(conf != null, "Configuration needs to be
set to init HoodieTableMetaClient");
+ ValidationUtils.checkArgument(conf != null || storage != null,
+ "Storage configuration or HoodieStorage needs to be set to init
HoodieTableMetaClient");
ValidationUtils.checkArgument(basePath != null, "basePath needs to be
set to init HoodieTableMetaClient");
- return newMetaClient(conf, basePath,
+ if (storage == null) {
+ storage = getStorage(new StoragePath(basePath), conf,
consistencyGuardConfig, fileSystemRetryConfig);
+ }
+ return newMetaClient(storage, basePath,
loadActiveTimelineOnLoad, consistencyGuardConfig, layoutVersion,
payloadClassName,
recordMergerStrategy, fileSystemRetryConfig, metaserverConfig);
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
index 66d96e8bfea..058320a32ae 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordReader.java
@@ -65,7 +65,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
-import static
org.apache.hudi.common.table.log.block.HoodieCommandBlock.HoodieCommandBlockTypeEnum.ROLLBACK_BLOCK;
import static
org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.COMPACTED_BLOCK_TIMES;
import static
org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.INSTANT_TIME;
import static
org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.TARGET_INSTANT_TIME;
@@ -160,7 +159,8 @@ public abstract class AbstractHoodieLogRecordReader {
this.latestInstantTime = latestInstantTime;
this.hoodieTableMetaClient = hoodieTableMetaClientOption.orElseGet(
() -> HoodieTableMetaClient.builder()
-
.setConf(storage.getConf().newInstance()).setBasePath(basePath).build());
+ .setStorage(storage)
+ .setBasePath(basePath).build());
// load class from the payload fully qualified class name
HoodieTableConfig tableConfig =
this.hoodieTableMetaClient.getTableConfig();
this.payloadClassFQN = tableConfig.getPayloadClass();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
index d34952aa0c8..00af75a2371 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
@@ -101,7 +101,8 @@ public class FileSystemViewManager {
*/
public SyncableFileSystemView getFileSystemView(String basePath) {
return globalViewMap.computeIfAbsent(basePath, (path) -> {
- HoodieTableMetaClient metaClient =
HoodieTableMetaClient.builder().setConf(conf.newInstance()).setBasePath(path).build();
+ HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
+ .setConf(conf.newInstance()).setBasePath(path).build();
return viewCreator.apply(metaClient, viewStorageConfig);
});
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
index 5737e2dcec0..9d905a09c77 100644
---
a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
+++
b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
@@ -71,7 +71,9 @@ public class FileBasedInternalSchemaStorageManager extends
AbstractInternalSchem
// make metaClient build lazy
private HoodieTableMetaClient getMetaClient() {
if (metaClient == null) {
- metaClient =
HoodieTableMetaClient.builder().setBasePath(baseSchemaPath.getParent().getParent().toString()).setConf(storage.getConf().newInstance()).build();
+ metaClient =
HoodieTableMetaClient.builder().setBasePath(baseSchemaPath.getParent().getParent().toString())
+ .setStorage(storage)
+ .build();
}
return metaClient;
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
index c3bd5c636c0..254f421284f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
@@ -28,8 +28,8 @@ import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
@@ -90,7 +90,7 @@ public abstract class BaseTableMetadata extends
AbstractHoodieTableMetadata {
super(engineContext, storage, dataBasePath);
this.dataMetaClient = HoodieTableMetaClient.builder()
- .setConf(storage.getConf().newInstance())
+ .setStorage(storage)
.setBasePath(dataBasePath)
.build();
diff --git
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
index 185791bbbec..31e44b9e212 100644
---
a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
+++
b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
@@ -128,7 +128,10 @@ public class HoodieBackedTableMetadata extends
BaseTableMetadata {
}
} else if (this.metadataMetaClient == null) {
try {
- this.metadataMetaClient =
HoodieTableMetaClient.builder().setConf(getStorageConf().newInstance()).setBasePath(metadataBasePath).build();
+ this.metadataMetaClient = HoodieTableMetaClient.builder()
+ .setStorage(storage)
+ .setBasePath(metadataBasePath)
+ .build();
this.metadataFileSystemView = getFileSystemView(metadataMetaClient);
this.metadataTableConfig = metadataMetaClient.getTableConfig();
} catch (TableNotFoundException e) {
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
index 56b2893a2cc..055e76f9e2b 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
@@ -33,7 +33,7 @@ import
org.apache.hudi.metaserver.client.HoodieMetaserverClient;
import org.apache.hudi.metaserver.client.HoodieMetaserverClientProxy;
import org.apache.hudi.metaserver.thrift.NoSuchObjectException;
import org.apache.hudi.metaserver.thrift.Table;
-import org.apache.hudi.storage.StorageConfiguration;
+import org.apache.hudi.storage.HoodieStorage;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -58,10 +58,10 @@ public class HoodieTableMetaserverClient extends
HoodieTableMetaClient {
private final Table table;
private final transient HoodieMetaserverClient metaserverClient;
- public HoodieTableMetaserverClient(StorageConfiguration<?> conf, String
basePath, ConsistencyGuardConfig consistencyGuardConfig,
+ public HoodieTableMetaserverClient(HoodieStorage storage, String basePath,
ConsistencyGuardConfig consistencyGuardConfig,
String mergerStrategy,
FileSystemRetryConfig fileSystemRetryConfig,
Option<String> databaseName,
Option<String> tableName, HoodieMetaserverConfig config) {
- super(conf, basePath, false, consistencyGuardConfig,
Option.of(TimelineLayoutVersion.CURR_LAYOUT_VERSION),
+ super(storage, basePath, false, consistencyGuardConfig,
Option.of(TimelineLayoutVersion.CURR_LAYOUT_VERSION),
config.getString(HoodieTableConfig.PAYLOAD_CLASS_NAME),
mergerStrategy, fileSystemRetryConfig);
this.databaseName = databaseName.isPresent() ? databaseName.get() :
tableConfig.getDatabaseName();
this.tableName = tableName.isPresent() ? tableName.get() :
tableConfig.getTableName();
