This is an automated email from the ASF dual-hosted git repository.
ahmedabualsaud pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9888900b5db Include Iceberg Hive runtime dependencies in IO expansion
service container (#32232)
9888900b5db is described below
commit 9888900b5db6f395984cb94a35ac415e4e43b319
Author: Ahmed Abualsaud <[email protected]>
AuthorDate: Mon Aug 19 13:19:49 2024 -0700
Include Iceberg Hive runtime dependencies in IO expansion service container
(#32232)
* include iceberg hive runtime dependencies in io expansion service
container
* trigger integration tests
---
.github/trigger_files/IO_Iceberg_Integration_Tests.json | 2 +-
sdks/java/io/expansion-service/build.gradle | 7 ++++++-
sdks/java/io/iceberg/hive/build.gradle | 4 ++--
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
index 62ae7886c57..2160d3c6800 100644
--- a/.github/trigger_files/IO_Iceberg_Integration_Tests.json
+++ b/.github/trigger_files/IO_Iceberg_Integration_Tests.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 4
+ "modification": 5
}
diff --git a/sdks/java/io/expansion-service/build.gradle
b/sdks/java/io/expansion-service/build.gradle
index 95a843e51fd..498950b3dc4 100644
--- a/sdks/java/io/expansion-service/build.gradle
+++ b/sdks/java/io/expansion-service/build.gradle
@@ -44,9 +44,14 @@ dependencies {
implementation project(":sdks:java:io:kafka:upgrade")
permitUnusedDeclared project(":sdks:java:io:kafka:upgrade") // BEAM-11761
- // Needed by Iceberg I/O users that use GCS for the warehouse location.
+ // **** IcebergIO runtime dependencies ****
+ runtimeOnly library.java.hadoop_client
+ // Needed when using GCS as the warehouse location.
implementation library.java.bigdataoss_gcs_connector
permitUnusedDeclared library.java.bigdataoss_gcs_connector
+ // Needed for HiveCatalog
+ runtimeOnly ("org.apache.iceberg:iceberg-hive-metastore:1.4.2")
+ runtimeOnly project(path: ":sdks:java:io:iceberg:hive:exec", configuration:
"shadow")
runtimeOnly library.java.kafka_clients
runtimeOnly library.java.slf4j_jdk14
diff --git a/sdks/java/io/iceberg/hive/build.gradle
b/sdks/java/io/iceberg/hive/build.gradle
index b81867ec90c..bfa6c75251c 100644
--- a/sdks/java/io/iceberg/hive/build.gradle
+++ b/sdks/java/io/iceberg/hive/build.gradle
@@ -34,8 +34,6 @@ dependencies {
// dependencies needed to run with iceberg's hive catalog
runtimeOnly ("org.apache.iceberg:iceberg-hive-metastore:$iceberg_version")
runtimeOnly project(path: ":sdks:java:io:iceberg:hive:exec",
configuration: "shadow")
- runtimeOnly library.java.bigdataoss_gcs_connector
- runtimeOnly library.java.hadoop_client
// ----- below dependencies are for testing and will not appear in the
shaded jar -----
// Beam IcebergIO dependencies
@@ -44,6 +42,8 @@ dependencies {
testImplementation project(":sdks:java:io:iceberg")
testRuntimeOnly project(path: ":runners:direct-java", configuration:
"shadow")
testRuntimeOnly library.java.snake_yaml
+ testRuntimeOnly library.java.bigdataoss_gcs_connector
+ testRuntimeOnly library.java.hadoop_client
// needed to set up the test environment
testImplementation "org.apache.iceberg:iceberg-common:$iceberg_version"
