This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 749071f9fe [#9046] feat(iceberg):Upgrade iceberg 1.10.0 for support
scan planning. (#9049)
749071f9fe is described below
commit 749071f9fea9ccaeff9d9a9bf063a2887d9cee19
Author: Xiaojian Sun <[email protected]>
AuthorDate: Thu Nov 13 22:22:51 2025 +0800
[#9046] feat(iceberg):Upgrade iceberg 1.10.0 for support scan planning.
(#9049)
### What changes in the PR?
Upgrade iceberg 1.10.0 for support scan planning.
### Why are the changes needed?
Fix: #([9046](https://github.com/apache/gravitino/issues/9046))
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
N/A
---
.../catalog-lakehouse-iceberg/build.gradle.kts | 5 +++++
dev/docker/gravitino/gravitino-dependency.sh | 2 +-
.../iceberg-rest-server-dependency.sh | 2 +-
docs/iceberg-rest-service.md | 10 +++++-----
docs/lakehouse-iceberg-catalog.md | 20 +++++++++----------
gradle/libs.versions.toml | 2 +-
iceberg/iceberg-common/build.gradle.kts | 23 +++++-----------------
iceberg/iceberg-rest-server/build.gradle.kts | 5 +++++
.../trino-test-tools/download_jar.sh | 4 ++--
9 files changed, 35 insertions(+), 38 deletions(-)
diff --git a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
index 25f750a6be..e61649bb69 100644
--- a/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-iceberg/build.gradle.kts
@@ -85,6 +85,11 @@ dependencies {
testImplementation(libs.testcontainers.mysql)
testImplementation(libs.testcontainers.postgresql)
+ // Add Hadoop 3.3+ dependencies since Spark's Hadoop is excluded
+ // Required for Iceberg 1.10+ which uses newer Hadoop APIs like
FileSystem.openFile()
+ testImplementation(libs.hadoop3.client.api)
+ testImplementation(libs.hadoop3.client.runtime)
+
testRuntimeOnly(libs.junit.jupiter.engine)
}
diff --git a/dev/docker/gravitino/gravitino-dependency.sh
b/dev/docker/gravitino/gravitino-dependency.sh
index e72551a059..aab5bcec7c 100755
--- a/dev/docker/gravitino/gravitino-dependency.sh
+++ b/dev/docker/gravitino/gravitino-dependency.sh
@@ -95,7 +95,7 @@ cp "${gravitino_staging_dir}/${pg_driver}"
"${gravitino_package_dir}/catalogs/jd
cp "${gravitino_staging_dir}/${pg_driver}"
"${gravitino_package_dir}/catalogs/lakehouse-iceberg/libs/"
cp "${gravitino_staging_dir}/${pg_driver}" "${gravitino_iceberg_rest_dir}"
-iceberg_version="1.9.2"
+iceberg_version="1.10.0"
iceberg_aws_bundle="iceberg-aws-bundle-${iceberg_version}.jar"
wget
"https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${iceberg_version}/${iceberg_aws_bundle}";
-O "${gravitino_staging_dir}/${iceberg_aws_bundle}"
cp "${gravitino_staging_dir}/${iceberg_aws_bundle}"
"${gravitino_iceberg_rest_dir}"
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
index 9ddc1b435f..e0e2000d1f 100755
--- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
+++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -63,7 +63,7 @@ find ${gravitino_home}/bundles/aws/build/libs/ -name
'gravitino-aws-*.jar' ! -na
find ${gravitino_home}/bundles/azure/build/libs/ -name 'gravitino-azure-*.jar'
! -name '*-empty.jar' -exec cp -v {} bundles/ \;
find ${gravitino_home}/bundles/aliyun/build/libs/ -name
'gravitino-aliyun-*.jar' ! -name '*-empty.jar' -exec cp -v {} bundles/ \;
-iceberg_version="1.9.2"
+iceberg_version="1.10.0"
iceberg_gcp_bundle="iceberg-gcp-bundle-${iceberg_version}.jar"
if [ ! -f "bundles/${iceberg_gcp_bundle}" ]; then
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index a15284f5b8..7d361177ef 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -19,7 +19,7 @@ There are some key difference between Gravitino Iceberg REST
server and Gravitin
### Capabilities
-- Supports the Apache Iceberg REST API defined in Iceberg 1.9, and supports
all namespace and table interfaces. The following interfaces are not
implemented yet:
+- Supports the Apache Iceberg REST API defined in Iceberg 1.10, and supports
all namespace and table interfaces. The following interfaces are not
implemented yet:
- multi table transaction
- pagination
- scan planning
@@ -250,7 +250,7 @@ Please refer the following configuration If you are using
Spark to access Iceber
```shell
./bin/spark-sql -v \
---conf
spark.jars=/Users/fanng/deploy/demo/jars/iceberg-spark-runtime-3.5_2.12-1.9.0.jar
\
+--conf
spark.jars=/Users/fanng/deploy/demo/jars/iceberg-spark-runtime-3.5_2.12-1.10.0.jar
\
--conf
spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
\
--conf spark.sql.catalog.rest=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.rest.rest.auth.type=oauth2 \
@@ -301,7 +301,7 @@ For other Iceberg s3 properties not managed by Gravitino
like `s3.sse.type`, you
Please refer to [S3
credentials](./security/credential-vending.md#s3-credentials) for credential
related configurations.
:::caution
-To resolve Log4j class conflict issues that may arise when using Iceberg AWS
1.9 bundle jars alongside the Gravitino Iceberg REST server, it is recommended
to remove the Log4j JAR files (specifically log4j-core and log4j-api) from the
`iceberg-rest-server/libs` directory.
+To resolve Log4j class conflict issues that may arise when using Iceberg AWS
1.10 bundle jars alongside the Gravitino Iceberg REST server, it is recommended
to remove the Log4j JAR files (specifically log4j-core and log4j-api) from the
`iceberg-rest-server/libs` directory.
:::
:::info
@@ -396,7 +396,7 @@ You could access the view interface if using JDBC backend
and enable `jdbc.schem
### Other Apache Iceberg catalog properties
-You can add other properties defined in [Iceberg catalog
properties](https://iceberg.apache.org/docs/1.9.2/configuration/#catalog-properties).
+You can add other properties defined in [Iceberg catalog
properties](https://iceberg.apache.org/docs/1.10.0/configuration/#catalog-properties).
The `clients` property for example:
| Configuration item | Description |
Default value | Required |
@@ -469,7 +469,7 @@ Normally you will see the output like
`{"defaults":{},"overrides":{}, "endpoints
### Deploying Apache Spark with Apache Iceberg support
-Follow the [Spark Iceberg start
guide](https://iceberg.apache.org/docs/1.9.2/spark-getting-started/) to set up
Apache Spark's and Apache Iceberg's environment.
+Follow the [Spark Iceberg start
guide](https://iceberg.apache.org/docs/1.10.0/spark-getting-started/) to set up
Apache Spark's and Apache Iceberg's environment.
### Starting the Apache Spark client with the Apache Iceberg REST catalog
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index 8875ebf78e..92564118f0 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -18,7 +18,7 @@ Apache Gravitino provides the ability to manage Apache
Iceberg metadata.
### Requirements and limitations
:::info
-Builds with Apache Iceberg `1.9.2`. The Apache Iceberg table format version is
`2` by default.
+Builds with Apache Iceberg `1.10.0`. The Apache Iceberg table format version
is `2` by default.
:::
## Catalog
@@ -124,7 +124,7 @@ Please make sure the credential file is accessible by
Gravitino, like using `exp
Please set `warehouse` to `gs://{bucket_name}/${prefix_name}`, and download
[Iceberg GCP bundle
jar](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-gcp-bundle)
and place it to `catalogs/lakehouse-iceberg/libs/`.
:::
-#### ADLS
+#### ADLS
Supports using Azure account name and secret key to access ADLS data.
@@ -208,13 +208,13 @@ Please refer to [Manage Relational Metadata Using
Gravitino](./manage-relational
Supports transforms:
- - `IdentityTransform`
- - `BucketTransform`
- - `TruncateTransform`
- - `YearTransform`
- - `MonthTransform`
- - `DayTransform`
- - `HourTransform`
+- `IdentityTransform`
+- `BucketTransform`
+- `TruncateTransform`
+- `YearTransform`
+- `MonthTransform`
+- `DayTransform`
+- `HourTransform`
:::info
Iceberg doesn't support multi fields in `BucketTransform`.
@@ -338,4 +338,4 @@ You can place `core-site.xml` and `hdfs-site.xml` in the
`catalogs/lakehouse-ice
:::info
Builds with Hadoop 2.10.x, there may be compatibility issues when accessing
Hadoop 3.x clusters.
When writing to HDFS, the Gravitino Iceberg REST server can only operate as
the specified HDFS user and doesn't support proxying to other HDFS users. See
[How to access Apache Hadoop](gravitino-server-config.md) for more details.
-:::
+:::
\ No newline at end of file
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index d0779e0ee6..1b357e543b 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -57,7 +57,7 @@ commons-collections3 = "3.2.2"
commons-configuration1 = "1.6"
commons-dbcp2 = "2.11.0"
caffeine = "2.9.3"
-iceberg = '1.9.2' # used for Gravitino Iceberg catalog and Iceberg REST service
+iceberg = '1.10.0' # used for Gravitino Iceberg catalog and Iceberg REST
service
iceberg4connector = "1.6.1" # used for compile connectors like Spark, Flink,
etc
paimon = '1.2.0'
spark33 = "3.3.4"
diff --git a/iceberg/iceberg-common/build.gradle.kts
b/iceberg/iceberg-common/build.gradle.kts
index bf83f34945..f9dc336976 100644
--- a/iceberg/iceberg-common/build.gradle.kts
+++ b/iceberg/iceberg-common/build.gradle.kts
@@ -47,25 +47,12 @@ dependencies {
implementation(libs.iceberg.azure)
implementation(libs.iceberg.hive.metastore)
implementation(libs.iceberg.gcp)
- implementation(libs.hadoop2.common) {
- exclude("com.github.spotbugs")
- exclude("com.sun.jersey")
- exclude("javax.servlet")
- exclude("org.apache.curator")
- exclude("org.apache.zookeeper")
- exclude("org.mortbay.jetty")
- }
+ // Upgrade to Hadoop 3.3+ for Iceberg 1.10 compatibility
+ // Iceberg 1.10 requires Hadoop 3.3+ APIs like FileSystem.openFile() and
FsTracer.get()
+ implementation(libs.hadoop3.client.api)
+ implementation(libs.hadoop3.client.runtime)
// use hdfs-default.xml
- implementation(libs.hadoop2.hdfs) {
- exclude("*")
- }
- implementation(libs.hadoop2.hdfs.client) {
- exclude("com.sun.jersey")
- exclude("javax.servlet")
- exclude("org.fusesource.leveldbjni")
- exclude("org.mortbay.jetty")
- }
- implementation(libs.hadoop2.mapreduce.client.core) {
+ implementation(libs.hadoop3.hdfs) {
exclude("*")
}
implementation(libs.hive2.metastore) {
diff --git a/iceberg/iceberg-rest-server/build.gradle.kts
b/iceberg/iceberg-rest-server/build.gradle.kts
index d183f4e0af..5cd60a4c8c 100644
--- a/iceberg/iceberg-rest-server/build.gradle.kts
+++ b/iceberg/iceberg-rest-server/build.gradle.kts
@@ -98,6 +98,11 @@ dependencies {
testImplementation(libs.slf4j.api)
testImplementation(libs.testcontainers)
+ // Add Hadoop 3.3+ dependencies since Spark's Hadoop is excluded
+ // Required for Iceberg 1.10+ which uses newer Hadoop APIs like
FileSystem.openFile()
+ testImplementation(libs.hadoop3.client.api)
+ testImplementation(libs.hadoop3.client.runtime)
+
testRuntimeOnly(libs.junit.jupiter.engine)
}
diff --git a/trino-connector/integration-test/trino-test-tools/download_jar.sh
b/trino-connector/integration-test/trino-test-tools/download_jar.sh
index 2c15cd7414..6f2ff2b932 100755
--- a/trino-connector/integration-test/trino-test-tools/download_jar.sh
+++ b/trino-connector/integration-test/trino-test-tools/download_jar.sh
@@ -61,8 +61,8 @@ download_postgresql_jar() {
}
download_iceberg_aws_bundle() {
- download_jar "iceberg-aws-bundle-1.9.2.jar" \
-
"https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.9.2/iceberg-aws-bundle-1.9.2.jar";
\
+ download_jar "iceberg-aws-bundle-1.10.0.jar" \
+
"https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.10.0/iceberg-aws-bundle-1.10.0.jar";
\
"$GRAVITINO_SERVER_DIR/catalogs/lakehouse-iceberg/libs"
}
