yihua commented on code in PR #11208:
URL: https://github.com/apache/hudi/pull/11208#discussion_r1599192983
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java:
##########
@@ -105,4 +109,4 @@ private static HoodieRowParquetWriteSupport
getHoodieRowParquetWriteSupport(Stor
StructType structType = HoodieInternalRowUtils.getCachedSchema(schema);
return
HoodieRowParquetWriteSupport.getHoodieRowParquetWriteSupport(conf.unwrapAs(Configuration.class),
structType, filter, config);
}
-}
+}
Review Comment:
nit: keep the new line
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java:
##########
@@ -20,30 +20,34 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.storage.StorageConfiguration;
/**
* Creates readers and writers for SPARK and AVRO record payloads
*/
public class HoodieSparkIOFactory extends HoodieHadoopIOFactory {
- private static final HoodieSparkIOFactory HOODIE_SPARK_IO_FACTORY = new
HoodieSparkIOFactory();
- public static HoodieSparkIOFactory getHoodieSparkIOFactory() {
- return HOODIE_SPARK_IO_FACTORY;
+ public HoodieSparkIOFactory(StorageConfiguration<?> storageConf) {
+ super(storageConf);
+ }
+
+ public static HoodieSparkIOFactory
getHoodieSparkIOFactory(StorageConfiguration<?> storageConf) {
+ return new HoodieSparkIOFactory(storageConf);
}
@Override
public HoodieFileReaderFactory
getReaderFactory(HoodieRecord.HoodieRecordType recordType) {
if (recordType == HoodieRecord.HoodieRecordType.SPARK) {
- return new HoodieSparkFileReaderFactory();
+ return new HoodieSparkFileReaderFactory(storageConf);
}
return super.getReaderFactory(recordType);
}
@Override
public HoodieFileWriterFactory
getWriterFactory(HoodieRecord.HoodieRecordType recordType) {
if (recordType == HoodieRecord.HoodieRecordType.SPARK) {
- return new HoodieSparkFileWriterFactory();
+ return new HoodieSparkFileWriterFactory(storageConf);
}
return super.getWriterFactory(recordType);
}
-}
+}
Review Comment:
Similar here for all files.
##########
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java:
##########
@@ -223,7 +222,8 @@ public static HoodieTableMetaClient
createMetaClient(StorageConfiguration<?> sto
*/
public static HoodieTableMetaClient createMetaClient(Configuration conf,
String basePath) {
- return createMetaClient(HoodieStorageUtils.getStorageConfWithCopy(conf),
basePath);
+ return createMetaClient((StorageConfiguration<Configuration>)
ReflectionUtils.loadClass(HADOOP_STORAGE_CONF,
Review Comment:
Should this static method be moved to `hudi-hadoop-common` and directly use
the constructor of `HadoopStorageConfiguration`?
##########
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieFileWriterFactory.java:
##########
@@ -120,4 +125,4 @@ public static boolean enableBloomFilter(boolean
populateMetaFields, HoodieConfig
// so the class HoodieIndexConfig cannot be accessed in hudi-common,
otherwise there will be a circular dependency problem
|| (config.contains("hoodie.index.type") &&
config.getString("hoodie.index.type").contains("BLOOM")));
}
-}
+}
Review Comment:
nit: keep new line.
##########
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieIOFactory.java:
##########
@@ -48,4 +48,13 @@ private static HoodieIOFactory getIOFactory(String
ioFactoryClass) {
public abstract HoodieFileWriterFactory
getWriterFactory(HoodieRecord.HoodieRecordType recordType);
+ public abstract HoodieStorage getStorage(StoragePath storagePath);
+
+ public abstract HoodieStorage getStorage(StoragePath path,
Review Comment:
How much effort is required to remove this API?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
