This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 74ef6601e [#6165] feat(core): Use Gravitino cloud jar without hadoop
packages for Iceberg REST server (#6168)
74ef6601e is described below
commit 74ef6601e465498877d273b1b9b58d0cef7aac6f
Author: FANNG <[email protected]>
AuthorDate: Fri Jan 10 15:12:42 2025 +0800
[#6165] feat(core): Use Gravitino cloud jar without hadoop packages for
Iceberg REST server (#6168)
### What changes were proposed in this pull request?
1. use Gravitino cloud jar without hadoop packages for Iceberg REST
server credential vending in test and document
2. For OSS, use Gravitino Aliyun bundle jar in test and docker image
because Iceberg doesn't provide Iceberg Aliyun bundle jar
### Why are the changes needed?
Fix: #6165
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
test S3 OSS GCS ADLS Iceberg REST test in local
---
.../iceberg-rest-server-dependency.sh | 18 +++++++-----------
docs/iceberg-rest-service.md | 8 +++++++-
docs/security/credential-vending.md | 21 ++++++++++++++++-----
iceberg/iceberg-rest-server/build.gradle.kts | 3 ++-
.../integration/test/IcebergRESTADLSTokenIT.java | 2 +-
.../test/IcebergRESTAzureAccountKeyIT.java | 2 +-
.../iceberg/integration/test/IcebergRESTGCSIT.java | 2 +-
.../iceberg/integration/test/IcebergRESTOSSIT.java | 2 ++
.../integration/test/IcebergRESTOSSSecretIT.java | 2 ++
...ebergRESTS3IT.java => IcebergRESTS3TokenIT.java} | 4 ++--
10 files changed, 41 insertions(+), 23 deletions(-)
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
index 852b55b02..aced0224f 100755
--- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
+++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -35,17 +35,18 @@ tar xfz gravitino-iceberg-rest-server-*.tar.gz
cp -r gravitino-iceberg-rest-server*-bin
${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server
cd ${gravitino_home}
-./gradlew :bundles:gcp-bundle:jar
-./gradlew :bundles:aws-bundle:jar
-./gradlew :bundles:azure-bundle:jar
+./gradlew :bundles:gcp:jar
+./gradlew :bundles:aws:jar
+./gradlew :bundles:azure:jar
+## Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun
bundle to provide OSS packages.
./gradlew :bundles:aliyun-bundle:jar
# prepare bundle jar
cd ${iceberg_rest_server_dir}
mkdir -p bundles
-cp ${gravitino_home}/bundles/gcp-bundle/build/libs/gravitino-gcp-bundle-*.jar
bundles/
-cp ${gravitino_home}/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*.jar
bundles/
-cp
${gravitino_home}/bundles/azure-bundle/build/libs/gravitino-azure-bundle-*.jar
bundles/
+cp ${gravitino_home}/bundles/gcp/build/libs/gravitino-gcp-*.jar bundles/
+cp ${gravitino_home}/bundles/aws/build/libs/gravitino-aws-*.jar bundles/
+cp ${gravitino_home}/bundles/azure/build/libs/gravitino-azure-*.jar bundles/
cp
${gravitino_home}/bundles/aliyun-bundle/build/libs/gravitino-aliyun-bundle-*.jar
bundles/
iceberg_gcp_bundle="iceberg-gcp-bundle-1.5.2.jar"
@@ -63,11 +64,6 @@ if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
curl -L -s -o bundles/${iceberg_azure_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/1.5.2/${iceberg_azure_bundle}
fi
-iceberg_aliyun_bundle="iceberg-aliyun-bundle-1.5.2.jar"
-if [ ! -f "bundles/${iceberg_aliyun_bundle}" ]; then
- curl -L -s -o bundles/${iceberg_aliyun_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aliyun-bundle/1.5.2/${iceberg_aliyun_bundle}
-fi
-
# download jdbc driver
curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar
https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index d42fc98b4..a4846d0e0 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -134,8 +134,14 @@ For other Iceberg OSS properties not managed by Gravitino
like `client.security-
Please refer to [OSS
credentials](./security/credential-vending.md#oss-credentials) for credential
related configurations.
+Additionally, Iceberg doesn't provide Iceberg Aliyun bundle jar which contains
OSS packages, there are two alternatives to use OSS packages:
+1. Use [Gravitino Aliyun bundle jar with hadoop
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle).
+2. Use [Aliyun JAVA
SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and
extract `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar`
jars.
+
+Please place the above jars in the classpath of Iceberg REST server, please
refer to [server management](#server-management) for classpath details.
+
:::info
-Please set the `gravitino.iceberg-rest.warehouse` parameter to
`oss://{bucket_name}/${prefix_name}`. Additionally, download the [Aliyun OSS
SDK](https://gosspublic.alicdn.com/sdks/java/aliyun_java_sdk_3.10.2.zip) and
copy `aliyun-sdk-oss-3.10.2.jar`, `hamcrest-core-1.1.jar`, `jdom2-2.0.6.jar` in
the classpath of Iceberg REST server, `iceberg-rest-server/libs` for the
auxiliary server, `libs` for the standalone server.
+Please set the `gravitino.iceberg-rest.warehouse` parameter to
`oss://{bucket_name}/${prefix_name}`.
:::
#### GCS
diff --git a/docs/security/credential-vending.md
b/docs/security/credential-vending.md
index 92370f431..b5391ac31 100644
--- a/docs/security/credential-vending.md
+++ b/docs/security/credential-vending.md
@@ -132,12 +132,23 @@ Gravitino supports custom credentials, you can implement
the `org.apache.graviti
Besides setting credentials related configuration, please download Gravitino
cloud bundle jar and place it in the classpath of Iceberg REST server or Hadoop
catalog.
-Gravitino cloud bundle jar:
+For Hadoop catalog, please use Gravitino cloud bundle jar with Hadoop and
cloud packages:
-- [Gravitino AWS bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws-bundle)
-- [Gravitino Aliyun bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle)
-- [Gravitino GCP bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp-bundle)
-- [Gravitino Azure bundle
jar](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure-bundle)
+- [Gravitino AWS bundle jar with Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws-bundle)
+- [Gravitino Aliyun bundle jar with Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle)
+- [Gravitino GCP bundle jar with Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp-bundle)
+- [Gravitino Azure bundle jar with Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure-bundle)
+
+For Iceberg REST catalog server, please use Gravitino cloud bundle jar without
Hadoop and cloud packages:
+
+- [Gravitino AWS bundle jar without Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws)
+- [Gravitino Aliyun bundle jar without Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun)
+- [Gravitino GCP bundle jar without Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-gcp)
+- [Gravitino Azure bundle jar without Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-azure)
+
+:::note
+For OSS, Iceberg doesn't provide Iceberg Aliyun bundle jar which contains OSS
packages, you could provide the OSS jar by yourself or use [Gravitino Aliyun
bundle jar with Hadoop and cloud
packages](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun-bundle),
please refer to [OSS
configuration](../iceberg-rest-service.md#oss-configuration) for more details.
+:::
The classpath of the server:
diff --git a/iceberg/iceberg-rest-server/build.gradle.kts
b/iceberg/iceberg-rest-server/build.gradle.kts
index fe35c4e77..925ad9007 100644
--- a/iceberg/iceberg-rest-server/build.gradle.kts
+++ b/iceberg/iceberg-rest-server/build.gradle.kts
@@ -62,7 +62,8 @@ dependencies {
annotationProcessor(libs.lombok)
compileOnly(libs.lombok)
- testImplementation(project(":bundles:aliyun"))
+ // Iceberg doesn't provide Aliyun bundle jar, use Gravitino Aliyun bundle to
provide OSS packages
+ testImplementation(project(":bundles:aliyun-bundle"))
testImplementation(project(":bundles:aws"))
testImplementation(project(":bundles:gcp", configuration = "shadow"))
testImplementation(project(":bundles:azure", configuration = "shadow"))
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
index 52ccb876d..bf718d601 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
@@ -130,7 +130,7 @@ public class IcebergRESTADLSTokenIT extends
IcebergRESTJdbcCatalogIT {
private void copyAzureBundleJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
- BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir);
+ BaseIT.copyBundleJarsToDirectory("azure", targetDir);
}
@Test
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
index f999f84f5..4f3c608fe 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
@@ -113,6 +113,6 @@ public class IcebergRESTAzureAccountKeyIT extends
IcebergRESTJdbcCatalogIT {
private void copyAzureBundleJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
- BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir);
+ BaseIT.copyBundleJarsToDirectory("azure", targetDir);
}
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
index 3396b60e1..74bf55edc 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTGCSIT.java
@@ -88,7 +88,7 @@ public class IcebergRESTGCSIT extends
IcebergRESTJdbcCatalogIT {
private void copyGCSBundleJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
- BaseIT.copyBundleJarsToDirectory("gcp-bundle", targetDir);
+ BaseIT.copyBundleJarsToDirectory("gcp", targetDir);
}
private void downloadIcebergBundleJar() throws IOException {
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
index 4c4b4a953..8e72ce33f 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSIT.java
@@ -126,6 +126,8 @@ public class IcebergRESTOSSIT extends
IcebergRESTJdbcCatalogIT {
private void copyAliyunOSSJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ // Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino
aliyun bundle to provide
+ // OSS packages.
BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
}
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
index 0be69cbe3..3b198c9d2 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTOSSSecretIT.java
@@ -111,6 +111,8 @@ public class IcebergRESTOSSSecretIT extends
IcebergRESTJdbcCatalogIT {
private void copyAliyunOSSJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
+ // Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino
aliyun bundle to provide
+ // OSS packages.
BaseIT.copyBundleJarsToDirectory("aliyun-bundle", targetDir);
}
}
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
similarity index 98%
rename from
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
rename to
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
index e906018f5..ef1551f91 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3IT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTS3TokenIT.java
@@ -40,7 +40,7 @@ import org.junit.platform.commons.util.StringUtils;
@SuppressWarnings("FormatStringAnnotation")
@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
-public class IcebergRESTS3IT extends IcebergRESTJdbcCatalogIT {
+public class IcebergRESTS3TokenIT extends IcebergRESTJdbcCatalogIT {
private String s3Warehouse;
private String accessKey;
@@ -124,7 +124,7 @@ public class IcebergRESTS3IT extends
IcebergRESTJdbcCatalogIT {
private void copyS3BundleJar() {
String gravitinoHome = System.getenv("GRAVITINO_HOME");
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
- BaseIT.copyBundleJarsToDirectory("aws-bundle", targetDir);
+ BaseIT.copyBundleJarsToDirectory("aws", targetDir);
}
/**
