This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 32a8b852ee1 [HUDI-6550] Add Hadoop conf to HiveConf for HiveSyncConfig
(#9221)
32a8b852ee1 is described below
commit 32a8b852ee12d3dfbb87abb0ea6a501b214ad6b9
Author: Shawn Chang <[email protected]>
AuthorDate: Wed Sep 13 18:26:34 2023 -0700
[HUDI-6550] Add Hadoop conf to HiveConf for HiveSyncConfig (#9221)
This commits fix the Hive sync config by creating new HiveConf object every
time when initializing HiveSyncConfig and adding hadoopConf as resource. We
have to load Hadoop conf otherwise properties like `--conf
spark.hadoop.hive.metastore.client.factory.class=com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory`
won't be able to be passed via Spark Hudi job.
Co-authored-by: Shawn Chang <[email protected]>
---
.../src/main/java/org/apache/hudi/hive/HiveSyncConfig.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java
index cf9274d6910..73f25b1615f 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java
@@ -98,8 +98,9 @@ public class HiveSyncConfig extends HoodieSyncConfig {
public HiveSyncConfig(Properties props, Configuration hadoopConf) {
super(props, hadoopConf);
- HiveConf hiveConf = hadoopConf instanceof HiveConf
- ? (HiveConf) hadoopConf : new HiveConf(hadoopConf, HiveConf.class);
+ HiveConf hiveConf = new HiveConf();
+ // HiveConf needs to load Hadoop conf to allow instantiation via
AWSGlueClientFactory
+ hiveConf.addResource(hadoopConf);
setHadoopConf(hiveConf);
validateParameters();
}
