This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b14f9e48d3d [HUDI-7005] Fix hudi-aws-bundle relocation issue with avro
(#9946)
b14f9e48d3d is described below
commit b14f9e48d3d81cb765e5b2fb355eb2c1e24ee582
Author: Prabhu Joseph <[email protected]>
AuthorDate: Sat Nov 4 11:04:06 2023 +0530
[HUDI-7005] Fix hudi-aws-bundle relocation issue with avro (#9946)
---
.../main/java/org/apache/hudi/sink/utils/HiveSyncContext.java | 9 +++++++--
packaging/hudi-aws-bundle/pom.xml | 2 --
packaging/hudi-flink-bundle/pom.xml | 1 -
3 files changed, 7 insertions(+), 5 deletions(-)
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java
index 0ea783feb5c..0d4cd10b8c7 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java
@@ -18,9 +18,9 @@
package org.apache.hudi.sink.utils;
-import org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool;
import org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.hive.HiveSyncTool;
@@ -67,6 +67,9 @@ public class HiveSyncContext {
private final Properties props;
private final HiveConf hiveConf;
+ public static final String AWS_GLUE_CATALOG_SYNC_TOOL_CLASS =
+ "org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool";
+
private HiveSyncContext(Properties props, HiveConf hiveConf) {
this.props = props;
this.hiveConf = hiveConf;
@@ -75,7 +78,9 @@ public class HiveSyncContext {
public HiveSyncTool hiveSyncTool() {
HiveSyncMode syncMode =
HiveSyncMode.of(props.getProperty(HIVE_SYNC_MODE.key()));
if (syncMode == HiveSyncMode.GLUE) {
- return new AwsGlueCatalogSyncTool(props, hiveConf);
+ return ((HiveSyncTool)
ReflectionUtils.loadClass(AWS_GLUE_CATALOG_SYNC_TOOL_CLASS,
+ new Class<?>[] {Properties.class,
org.apache.hadoop.conf.Configuration.class},
+ props, hiveConf));
}
return new HiveSyncTool(props, hiveConf);
}
diff --git a/packaging/hudi-aws-bundle/pom.xml
b/packaging/hudi-aws-bundle/pom.xml
index 6d2596f49a3..1250296cceb 100644
--- a/packaging/hudi-aws-bundle/pom.xml
+++ b/packaging/hudi-aws-bundle/pom.xml
@@ -72,12 +72,10 @@
</transformers>
<artifactSet>
<includes combine.children="append">
-
<include>org.apache.hudi:hudi-common</include>
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-hive-sync</include>
<include>org.apache.hudi:hudi-aws</include>
-
<include>org.apache.parquet:parquet-avro</include>
<include>org.reactivestreams:reactive-streams</include>
<include>com.amazonaws:dynamodb-lock-client</include>
<include>org.apache.httpcomponents:httpclient</include>
diff --git a/packaging/hudi-flink-bundle/pom.xml
b/packaging/hudi-flink-bundle/pom.xml
index 4a490c43a98..e141e975232 100644
--- a/packaging/hudi-flink-bundle/pom.xml
+++ b/packaging/hudi-flink-bundle/pom.xml
@@ -84,7 +84,6 @@
<include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-timeline-service</include>
- <include>org.apache.hudi:hudi-aws</include>
<!-- Kryo -->
<include>com.esotericsoftware:kryo-shaded</include>
