yihua commented on code in PR #11131:
URL: https://github.com/apache/hudi/pull/11131#discussion_r1595862165
##########
hudi-common/src/main/java/org/apache/hudi/storage/HoodieStorageUtils.java:
##########
@@ -19,37 +19,63 @@
package org.apache.hudi.storage;
-import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
-import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
+import org.apache.hudi.common.fs.ConsistencyGuard;
+import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
public class HoodieStorageUtils {
+
+ public static final String HUDI_HADOOP_STORAGE =
"org.apache.hudi.storage.hadoop.HoodieHadoopStorage";
+ public static final String HADOOP_STORAGE_CONF =
"org.apache.hudi.storage.hadoop.HadoopStorageConfiguration";
public static final String DEFAULT_URI = "file:///";
public static HoodieStorage getStorage(StorageConfiguration<?> conf) {
return getStorage(DEFAULT_URI, conf);
}
public static HoodieStorage getStorage(FileSystem fs) {
- return new HoodieHadoopStorage(fs);
+ return (HoodieStorage) ReflectionUtils.loadClass(HUDI_HADOOP_STORAGE, new
Class<?>[] {FileSystem.class}, fs);
}
public static HoodieStorage getStorage(String basePath,
StorageConfiguration<?> conf) {
- return getStorage(HadoopFSUtils.getFs(basePath, conf));
+ return (HoodieStorage) ReflectionUtils.loadClass(HUDI_HADOOP_STORAGE, new
Class<?>[] {String.class, StorageConfiguration.class}, basePath, conf);
+ }
+
+ public static HoodieStorage getStorage(String basePath, Configuration conf) {
+ return (HoodieStorage) ReflectionUtils.loadClass(HUDI_HADOOP_STORAGE, new
Class<?>[] {String.class, Configuration.class}, basePath, conf);
}
public static HoodieStorage getStorage(StoragePath path,
StorageConfiguration<?> conf) {
- return getStorage(HadoopFSUtils.getFs(path,
conf.unwrapAs(Configuration.class)));
+ return (HoodieStorage) ReflectionUtils.loadClass(HUDI_HADOOP_STORAGE, new
Class<?>[] {StoragePath.class, StorageConfiguration.class}, path, conf);
+ }
+
+ public static HoodieStorage getStorage(StoragePath path,
+ StorageConfiguration<?> conf,
+ boolean enableRetry,
+ long maxRetryIntervalMs,
+ int maxRetryNumbers,
+ long initialRetryIntervalMs,
+ String retryExceptions,
+ ConsistencyGuard consistencyGuard) {
+ return (HoodieStorage) ReflectionUtils.loadClass(HUDI_HADOOP_STORAGE,
+ new Class<?>[] {StoragePath.class, StorageConfiguration.class,
boolean.class, long.class, int.class, long.class,
+ String.class, ConsistencyGuard.class},
+ path, conf, enableRetry, maxRetryIntervalMs, maxRetryNumbers,
initialRetryIntervalMs, retryExceptions,
+ consistencyGuard);
}
public static HoodieStorage getRawStorage(HoodieStorage storage) {
- FileSystem fs = (FileSystem) storage.getFileSystem();
- if (fs instanceof HoodieWrapperFileSystem) {
- return getStorage(((HoodieWrapperFileSystem) fs).getFileSystem());
- }
- return storage;
+ return (HoodieStorage) ReflectionUtils.loadClass(HUDI_HADOOP_STORAGE, new
Class<?>[] {HoodieStorage.class}, storage);
+ }
+
+ public static StorageConfiguration<?> getStorageConf(Configuration conf) {
+ return (StorageConfiguration<?>)
ReflectionUtils.loadClass(HADOOP_STORAGE_CONF,
+ new Class<?>[] {Configuration.class}, conf);
+ }
+
+ public static StorageConfiguration<Configuration>
getStorageConfWithCopy(Configuration conf) {
+ return (StorageConfiguration<Configuration>)
getStorageConf(conf).newInstance();
Review Comment:
Should this use the constructor `HadoopStorageConfiguration(Configuration
configuration, boolean copy)` instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
