This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 829303be669 [HUDI-6622] Reuse the table config from
HoodieTableMetaClient in the HoodieTableMetaserverClient (#9330)
829303be669 is described below
commit 829303be669d38ec4cfa5d613829e8b2e8e2ebea
Author: Dongsj <[email protected]>
AuthorDate: Fri Aug 4 15:10:05 2023 +0800
[HUDI-6622] Reuse the table config from HoodieTableMetaClient in the
HoodieTableMetaserverClient (#9330)
---
.../hudi/common/table/HoodieTableMetaClient.java | 43 +++++++++++-----------
.../common/table/HoodieTableMetaserverClient.java | 25 ++++++-------
2 files changed, 33 insertions(+), 35 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
index afb590e9cc0..07ff54b2ed2 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
@@ -120,14 +120,12 @@ public class HoodieTableMetaClient implements
Serializable {
protected HoodieMetaserverConfig metaserverConfig;
/**
- *
* Instantiate HoodieTableMetaClient.
* Can only be called if table already exists
- *
*/
protected HoodieTableMetaClient(Configuration conf, String basePath, boolean
loadActiveTimelineOnLoad,
- ConsistencyGuardConfig consistencyGuardConfig,
Option<TimelineLayoutVersion> layoutVersion,
- String payloadClassName, String
recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig) {
+ ConsistencyGuardConfig
consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
+ String payloadClassName, String
recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig) {
LOG.info("Loading HoodieTableMetaClient from " + basePath);
this.consistencyGuardConfig = consistencyGuardConfig;
this.fileSystemRetryConfig = fileSystemRetryConfig;
@@ -243,7 +241,7 @@ public class HoodieTableMetaClient implements Serializable {
/**
* Returns Marker folder path.
- *
+ *
* @param instantTs Instant Timestamp
* @return
*/
@@ -328,7 +326,7 @@ public class HoodieTableMetaClient implements Serializable {
/**
* Return raw file-system.
- *
+ *
* @return fs
*/
public FileSystem getRawFs() {
@@ -471,7 +469,7 @@ public class HoodieTableMetaClient implements Serializable {
* @return Instance of HoodieTableMetaClient
*/
public static HoodieTableMetaClient initTableAndGetMetaClient(Configuration
hadoopConf, String basePath,
- Properties props) throws IOException {
+ Properties
props) throws IOException {
LOG.info("Initializing " + basePath + " as hoodie table " + basePath);
Path basePathDir = new Path(basePath);
final FileSystem fs = FSUtils.getFs(basePath, hadoopConf);
@@ -540,8 +538,8 @@ public class HoodieTableMetaClient implements Serializable {
/**
* Helper method to scan all hoodie-instant metafiles.
*
- * @param fs The file system implementation for this table
- * @param metaPath The meta path where meta files are stored
+ * @param fs The file system implementation for this table
+ * @param metaPath The meta path where meta files are stored
* @param nameFilter The name filter to filter meta files
* @return An array of meta FileStatus
* @throws IOException In case of failure
@@ -615,29 +613,29 @@ public class HoodieTableMetaClient implements
Serializable {
/**
* Helper method to scan all hoodie-instant metafiles and construct
HoodieInstant objects.
*
- * @param includedExtensions Included hoodie extensions
+ * @param includedExtensions Included hoodie extensions
* @param applyLayoutVersionFilters Depending on Timeline layout version, if
there are multiple states for the same
- * action instant, only include the highest state
+ * action instant, only include the highest
state
* @return List of Hoodie Instants generated
* @throws IOException in case of failure
*/
public List<HoodieInstant> scanHoodieInstantsFromFileSystem(Set<String>
includedExtensions,
- boolean applyLayoutVersionFilters) throws IOException {
+ boolean
applyLayoutVersionFilters) throws IOException {
return scanHoodieInstantsFromFileSystem(metaPath.get(),
includedExtensions, applyLayoutVersionFilters);
}
/**
* Helper method to scan all hoodie-instant metafiles and construct
HoodieInstant objects.
*
- * @param timelinePath MetaPath where instant files are stored
- * @param includedExtensions Included hoodie extensions
+ * @param timelinePath MetaPath where instant files are stored
+ * @param includedExtensions Included hoodie extensions
* @param applyLayoutVersionFilters Depending on Timeline layout version, if
there are multiple states for the same
- * action instant, only include the highest state
+ * action instant, only include the highest
state
* @return List of Hoodie Instants generated
* @throws IOException in case of failure
*/
public List<HoodieInstant> scanHoodieInstantsFromFileSystem(Path
timelinePath, Set<String> includedExtensions,
- boolean applyLayoutVersionFilters) throws IOException {
+ boolean
applyLayoutVersionFilters) throws IOException {
Stream<HoodieInstant> instantStream = Arrays.stream(
HoodieTableMetaClient
.scanFiles(getFs(), timelinePath, path -> {
@@ -684,13 +682,14 @@ public class HoodieTableMetaClient implements
Serializable {
}
private static HoodieTableMetaClient newMetaClient(Configuration conf,
String basePath, boolean loadActiveTimelineOnLoad,
- ConsistencyGuardConfig consistencyGuardConfig,
Option<TimelineLayoutVersion> layoutVersion,
- String payloadClassName, String recordMergerStrategy,
FileSystemRetryConfig fileSystemRetryConfig, HoodieMetaserverConfig
metaserverConfig) {
+ ConsistencyGuardConfig
consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
+ String payloadClassName,
String recordMergerStrategy, FileSystemRetryConfig fileSystemRetryConfig,
HoodieMetaserverConfig metaserverConfig) {
return metaserverConfig.isMetaserverEnabled()
? (HoodieTableMetaClient)
ReflectionUtils.loadClass("org.apache.hudi.common.table.HoodieTableMetaserverClient",
- new Class<?>[]{Configuration.class, String.class,
ConsistencyGuardConfig.class, String.class, FileSystemRetryConfig.class,
String.class, String.class, HoodieMetaserverConfig.class},
+ new Class<?>[] {Configuration.class, String.class,
ConsistencyGuardConfig.class, String.class,
+ FileSystemRetryConfig.class, Option.class, Option.class,
HoodieMetaserverConfig.class},
conf, basePath, consistencyGuardConfig, recordMergerStrategy,
fileSystemRetryConfig,
- metaserverConfig.getDatabaseName(), metaserverConfig.getTableName(),
metaserverConfig)
+ Option.ofNullable(metaserverConfig.getDatabaseName()),
Option.ofNullable(metaserverConfig.getTableName()), metaserverConfig)
: new HoodieTableMetaClient(conf, basePath,
loadActiveTimelineOnLoad, consistencyGuardConfig, layoutVersion,
payloadClassName, recordMergerStrategy, fileSystemRetryConfig);
}
@@ -991,7 +990,7 @@ public class HoodieTableMetaClient implements Serializable {
public PropertyBuilder fromProperties(Properties properties) {
HoodieConfig hoodieConfig = new HoodieConfig(properties);
- for (String key: HoodieTableConfig.PERSISTED_CONFIG_LIST) {
+ for (String key : HoodieTableConfig.PERSISTED_CONFIG_LIST) {
Object value = hoodieConfig.getString(key);
if (value != null) {
set(key, value);
@@ -1193,7 +1192,7 @@ public class HoodieTableMetaClient implements
Serializable {
* Init Table with the properties build by this builder.
*
* @param configuration The hadoop config.
- * @param basePath The base path for hoodie table.
+ * @param basePath The base path for hoodie table.
*/
public HoodieTableMetaClient initTable(Configuration configuration, String
basePath)
throws IOException {
diff --git
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
index 85830e5ab14..85e89d75eb5 100644
---
a/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
+++
b/hudi-platform-service/hudi-metaserver/hudi-metaserver-client/src/main/java/org/apache/hudi/common/table/HoodieTableMetaserverClient.java
@@ -60,23 +60,22 @@ public class HoodieTableMetaserverClient extends
HoodieTableMetaClient {
public HoodieTableMetaserverClient(Configuration conf, String basePath,
ConsistencyGuardConfig consistencyGuardConfig,
String mergerStrategy,
FileSystemRetryConfig fileSystemRetryConfig,
- String databaseName, String tableName,
HoodieMetaserverConfig config) {
+ Option<String> databaseName,
Option<String> tableName, HoodieMetaserverConfig config) {
super(conf, basePath, false, consistencyGuardConfig,
Option.of(TimelineLayoutVersion.CURR_LAYOUT_VERSION),
config.getString(HoodieTableConfig.PAYLOAD_CLASS_NAME),
mergerStrategy, fileSystemRetryConfig);
- checkArgument(nonEmpty(databaseName), "database name is required.");
- checkArgument(nonEmpty(tableName), "table name is required.");
- this.databaseName = databaseName;
- this.tableName = tableName;
+ this.databaseName = databaseName.isPresent() ? databaseName.get() :
tableConfig.getDatabaseName();
+ this.tableName = tableName.isPresent() ? tableName.get() :
tableConfig.getTableName();
this.metaserverConfig = config;
this.metaserverClient = HoodieMetaserverClientProxy.getProxy(config);
- this.table = initOrGetTable(databaseName, tableName, config);
+ this.table = initOrGetTable(config);
// TODO: transfer table parameters to table config
- this.tableConfig = new HoodieTableConfig();
tableConfig.setTableVersion(HoodieTableVersion.current());
tableConfig.setAll(config.getProps());
}
- private Table initOrGetTable(String db, String tb, HoodieMetaserverConfig
config) {
+ private Table initOrGetTable(HoodieMetaserverConfig config) {
+ checkArgument(nonEmpty(databaseName), "database name is required.");
+ checkArgument(nonEmpty(tableName), "table name is required.");
Table table;
try {
table = metaserverClient.getTable(databaseName, tableName);
@@ -88,10 +87,10 @@ public class HoodieTableMetaserverClient extends
HoodieTableMetaClient {
} catch (IOException ioException) {
LOG.info("Failed to get the user", ioException);
}
- LOG.info(String.format("Table %s.%s doesn't exist, will create it.",
db, tb));
+ LOG.info(String.format("Table %s.%s doesn't exist, will create it.",
databaseName, tableName));
table = new Table();
- table.setDatabaseName(db);
- table.setTableName(tb);
+ table.setDatabaseName(databaseName);
+ table.setTableName(tableName);
table.setLocation(config.getString(HoodieWriteConfig.BASE_PATH));
table.setOwner(user);
table.setTableType(config.getString(HoodieTableConfig.TYPE.key()));
@@ -134,12 +133,12 @@ public class HoodieTableMetaserverClient extends
HoodieTableMetaClient {
}
public List<HoodieInstant> scanHoodieInstantsFromFileSystem(Set<String>
includedExtensions,
- boolean applyLayoutVersionFilters) {
+ boolean
applyLayoutVersionFilters) {
throw new HoodieException("Unsupport operation");
}
public List<HoodieInstant> scanHoodieInstantsFromFileSystem(Path
timelinePath, Set<String> includedExtensions,
- boolean applyLayoutVersionFilters) {
+ boolean
applyLayoutVersionFilters) {
throw new HoodieException("Unsupport operation");
}
