This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new b1e5931b27 [#6394] feat(iceberg): upgrade Iceberg version from 1.5.2
to 1.6.1 (#6374)
b1e5931b27 is described below
commit b1e5931b27174a73bcc00921f392e0db02bba2d7
Author: FANNG <[email protected]>
AuthorDate: Fri Feb 7 14:26:10 2025 +0800
[#6394] feat(iceberg): upgrade Iceberg version from 1.5.2 to 1.6.1 (#6374)
### What changes were proposed in this pull request?
Upgrade Iceberg version from 1.5.2 to 1.6.1
In 1.6, the `token` endpoint is deprecated and there're no new endpoint
is added.
### Why are the changes needed?
Fix: #6394
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
run pass existing ITs
---
build.gradle.kts | 3 +++
.../iceberg-rest-server/iceberg-rest-server-dependency.sh | 14 ++++++++------
docs/iceberg-rest-service.md | 5 ++---
docs/lakehouse-iceberg-catalog.md | 2 +-
gradle/libs.versions.toml | 2 +-
.../iceberg/integration/test/IcebergRESTADLSTokenIT.java | 8 ++++----
.../integration/test/IcebergRESTAzureAccountKeyIT.java | 8 ++++----
.../iceberg/integration/test/IcebergRESTGCSIT.java | 8 +++++---
.../iceberg/integration/test/IcebergRESTOSSIT.java | 8 ++++----
.../iceberg/integration/test/IcebergRESTOSSSecretIT.java | 8 ++++----
.../iceberg/integration/test/IcebergRESTS3TokenIT.java | 8 ++++----
.../apache/gravitino/integration/test/util/ITUtils.java | 4 ++++
.../integration-test/trino-test-tools/download_jar.sh | 4 ++--
13 files changed, 46 insertions(+), 36 deletions(-)
diff --git a/build.gradle.kts b/build.gradle.kts
index 1fe3c80d9b..b56f6f4d79 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -191,6 +191,9 @@ allprojects {
param.environment("NEED_CREATE_DOCKER_NETWORK", "true")
}
+ val icebergVersion: String = libs.versions.iceberg.get()
+ param.systemProperty("ICEBERG_VERSION", icebergVersion)
+
// Change poll image pause time from 30s to 60s
param.environment("TESTCONTAINERS_PULL_PAUSE_TIMEOUT", "60")
val jdbcDatabase = project.properties["jdbcBackend"] as? String ?: "h2"
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
index aced0224f4..1640d0195b 100755
--- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
+++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -49,19 +49,21 @@ cp
${gravitino_home}/bundles/aws/build/libs/gravitino-aws-*.jar bundles/
cp ${gravitino_home}/bundles/azure/build/libs/gravitino-azure-*.jar bundles/
cp
${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar
bundles/
-iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar"
+iceberg_version="1.6.1"
+
+iceberg_gcp_bundle="iceberg-gcp-bundle-${iceberg_version}.jar"
if [ ! -f "bundles/${iceberg_gcp_bundle}" ]; then
- curl -L -s -o bundles/${iceberg_gcp_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/1.5.2/${iceberg_gcp_bundle}
+ curl -L -s -o bundles/${iceberg_gcp_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/${iceberg_version}/${iceberg_gcp_bundle}
fi
-iceberg_aws_bundle="iceberg-aws-bundle-1.5.2.jar"
+iceberg_aws_bundle="iceberg-aws-bundle-${iceberg_version}.jar"
if [ ! -f "bundles/${iceberg_aws_bundle}" ]; then
- curl -L -s -o bundles/${iceberg_aws_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/1.5.2/${iceberg_aws_bundle}
+ curl -L -s -o bundles/${iceberg_aws_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${iceberg_version}/${iceberg_aws_bundle}
fi
-iceberg_azure_bundle="iceberg-azure-bundle-1.5.2.jar"
+iceberg_azure_bundle="iceberg-azure-bundle-${iceberg_version}.jar"
if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
- curl -L -s -o bundles/${iceberg_azure_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle}
+ curl -L -s -o bundles/${iceberg_azure_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${iceberg_version}/${iceberg_azure_bundle}
fi
# download jdbc driver
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index a4846d0e0d..f89b6f2672 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -13,7 +13,6 @@ The Apache Gravitino Iceberg REST Server follows the [Apache
Iceberg REST API sp
### Capabilities
- Supports the Apache Iceberg REST API defined in Iceberg 1.5, and supports
all namespace and table interfaces. The following interfaces are not
implemented yet:
- - token
- multi table transaction
- pagination
- Works as a catalog proxy, supporting `Hive` and `JDBC` as catalog backend.
@@ -317,7 +316,7 @@ gravitino.iceberg-rest.gravitino-metalake = test
### Other Apache Iceberg catalog properties
-You can add other properties defined in [Iceberg catalog
properties](https://iceberg.apache.org/docs/1.5.2/configuration/#catalog-properties).
+You can add other properties defined in [Iceberg catalog
properties](https://iceberg.apache.org/docs/1.6.1/configuration/#catalog-properties).
The `clients` property for example:
| Configuration item | Description |
Default value | Required |
@@ -379,7 +378,7 @@ Normally you will see the output like
`{"defaults":{},"overrides":{}}%`.
### Deploying Apache Spark with Apache Iceberg support
-Follow the [Spark Iceberg start
guide](https://iceberg.apache.org/docs/1.5.2/spark-getting-started/) to set up
Apache Spark's and Apache Iceberg's environment.
+Follow the [Spark Iceberg start
guide](https://iceberg.apache.org/docs/1.6.1/spark-getting-started/) to set up
Apache Spark's and Apache Iceberg's environment.
### Starting the Apache Spark client with the Apache Iceberg REST catalog
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index f8b462b24e..8fb9c61f32 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -18,7 +18,7 @@ Apache Gravitino provides the ability to manage Apache
Iceberg metadata.
### Requirements and limitations
:::info
-Builds with Apache Iceberg `1.5.2`. The Apache Iceberg table format version is
`2` by default.
+Builds with Apache Iceberg `1.6.1`. The Apache Iceberg table format version is
`2` by default.
:::
## Catalog
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 767cbe0827..736691c4b5 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -55,7 +55,7 @@ commons-collections3 = "3.2.2"
commons-configuration1 = "1.6"
commons-dbcp2 = "2.11.0"
caffeine = "2.9.3"
-iceberg = '1.5.2' # used for Gravitino Iceberg catalog and Iceberg REST service
+iceberg = '1.6.1' # used for Gravitino Iceberg catalog and Iceberg REST service
iceberg4spark = "1.4.1" # used for compile spark connector
paimon = '0.8.0'
spark33 = "3.3.4"
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
index bf718d601d..b4c70a3829 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
@@ -117,11 +117,11 @@ public class IcebergRESTADLSTokenIT extends
IcebergRESTJdbcCatalogIT {
}
private void downloadIcebergAzureBundleJar() throws IOException {
- String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar";
String icebergBundleJarUri =
- "https://repo1.maven.org/maven2/org/apache/iceberg/";
- + "iceberg-azure-bundle/1.5.2/"
- + icebergBundleJarName;
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-azure-bundle/%s/iceberg-azure-bundle-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
index 4f3c608fe3..e74a8db677 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
@@ -100,11 +100,11 @@ public class IcebergRESTAzureAccountKeyIT extends
IcebergRESTJdbcCatalogIT {
}
private void downloadIcebergAzureBundleJar() throws IOException {
- String icebergBundleJarName = "iceberg-azure-bundle-1.5.2.jar";
String icebergBundleJarUri =
- "https://repo1.maven.org/maven2/org/apache/iceberg/";
- + "iceberg-azure-bundle/1.5.2/"
- + icebergBundleJarName;
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-azure-bundle/%s/iceberg-azure-bundle-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
index 74bf55edc0..1ada58189e 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
@@ -35,6 +35,7 @@ import
org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
// You should export GRAVITINO_GCS_BUCKET and GOOGLE_APPLICATION_CREDENTIALS
to run the test
@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
public class IcebergRESTGCSIT extends IcebergRESTJdbcCatalogIT {
+
private String gcsWarehouse;
private String gcsCredentialPath;
@@ -92,10 +93,11 @@ public class IcebergRESTGCSIT extends
IcebergRESTJdbcCatalogIT {
}
private void downloadIcebergBundleJar() throws IOException {
- String icebergBundleJarName = "iceberg-gcp-bundle-1.5.2.jar";
String icebergBundleJarUri =
-
"https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-gcp-bundle/1.5.2/";
- + icebergBundleJarName;
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-gcp-bundle/%s/iceberg-gcp-bundle-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
index 8e72ce33f1..a867071dda 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
@@ -113,11 +113,11 @@ public class IcebergRESTOSSIT extends
IcebergRESTJdbcCatalogIT {
}
private void downloadIcebergForAliyunJar() throws IOException {
- String icebergBundleJarName = "iceberg-aliyun-1.5.2.jar";
String icebergBundleJarUri =
- "https://repo1.maven.org/maven2/org/apache/iceberg/";
- + "iceberg-aliyun/1.5.2/"
- + icebergBundleJarName;
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-aliyun/%s/iceberg-aliyun-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
index 3b198c9d29..38239bb1d9 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
@@ -98,11 +98,11 @@ public class IcebergRESTOSSSecretIT extends
IcebergRESTJdbcCatalogIT {
}
private void downloadIcebergForAliyunJar() throws IOException {
- String icebergBundleJarName = "iceberg-aliyun-1.5.2.jar";
String icebergBundleJarUri =
- "https://repo1.maven.org/maven2/org/apache/iceberg/";
- + "iceberg-aliyun/1.5.2/"
- + icebergBundleJarName;
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-aliyun/%s/iceberg-aliyun-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
index ef1551f91b..989dce834c 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
@@ -111,11 +111,11 @@ public class IcebergRESTS3TokenIT extends
IcebergRESTJdbcCatalogIT {
}
private void downloadIcebergAwsBundleJar() throws IOException {
- String icebergBundleJarName = "iceberg-aws-bundle-1.5.2.jar";
String icebergBundleJarUri =
- "https://repo1.maven.org/maven2/org/apache/iceberg/";
- + "iceberg-aws-bundle/1.5.2/"
- + icebergBundleJarName;
+ String.format(
+ "https://repo1.maven.org/maven2/org/apache/iceberg/";
+ + "iceberg-aws-bundle/%s/iceberg-aws-bundle-%s.jar",
+ ITUtils.icebergVersion(), ITUtils.icebergVersion());
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
DownloaderUtils.downloadFile(icebergBundleJarUri, targetDir);
diff --git
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
index d7c099dc7a..508c5fb42e 100644
---
a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
+++
b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/ITUtils.java
@@ -187,6 +187,10 @@ public class ITUtils {
return Objects.equals(mode, ITUtils.EMBEDDED_TEST_MODE);
}
+ public static String icebergVersion() {
+ return System.getProperty("ICEBERG_VERSION");
+ }
+
public static String getBundleJarSourceFile(String bundleName) {
String jarName = ITUtils.getBundleJarName(bundleName);
String gcsJars =
ITUtils.joinPath(ITUtils.getBundleJarDirectory(bundleName), jarName);
diff --git a/trino-connector/integration-test/trino-test-tools/download_jar.sh
b/trino-connector/integration-test/trino-test-tools/download_jar.sh
index 36cba2bce9..3dcdf8803c 100755
--- a/trino-connector/integration-test/trino-test-tools/download_jar.sh
+++ b/trino-connector/integration-test/trino-test-tools/download_jar.sh
@@ -62,8 +62,8 @@ download_postgresql_jar() {
}
download_iceberg_aws_bundle() {
- download_jar "iceberg-aws-bundle--1.5.2.jar" \
- "https://jdbc.postgresql.org/download/iceberg-aws-bundle--1.5.2.jar"; \
+ download_jar "iceberg-aws-bundle-1.6.1.jar" \
+ "https://jdbc.postgresql.org/download/iceberg-aws-bundle-1.6.1.jar"; \
"$GRAVITINO_SERVER_DIR/catalogs/lakehouse-iceberg/libs"
}
