This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 8aad8ff2ad [#8733][#8704] [#8754] fix(iceberg): fix start Iceberg
REST server start failed in docker enviroment (#8739)
8aad8ff2ad is described below
commit 8aad8ff2ad3f10ce02b7b2ef79d7592be643906b
Author: FANNG <[email protected]>
AuthorDate: Sat Oct 11 16:15:02 2025 +0900
[#8733][#8704] [#8754] fix(iceberg): fix start Iceberg REST server start
failed in docker enviroment (#8739)
### What changes were proposed in this pull request?
1. remove log4j from gravitino Iceberg REST server docker image
2. fix cgroup errors in specific enviroment
3. not using the gravitino-aliyun-bundle jar to avoid hadoop class
conflict
### Why are the changes needed?
Fix: #8733
Fix: #8704
Fix: #8754
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
test IRC and Gravitino server in specific ubuntu machines
---
dev/docker/gravitino/gravitino-dependency.sh | 23 ++++++++++++++++-
dev/docker/gravitino/start-gravitino.sh | 5 +++-
.../iceberg-rest-server-dependency.sh | 30 +++++++++++++++++++---
.../start-iceberg-rest-server.sh | 3 +++
docs/iceberg-rest-service.md | 4 +++
docs/lakehouse-iceberg-catalog.md | 4 +++
6 files changed, 63 insertions(+), 6 deletions(-)
diff --git a/dev/docker/gravitino/gravitino-dependency.sh
b/dev/docker/gravitino/gravitino-dependency.sh
index d2f1c5d742..e72551a059 100755
--- a/dev/docker/gravitino/gravitino-dependency.sh
+++ b/dev/docker/gravitino/gravitino-dependency.sh
@@ -48,6 +48,21 @@ download_gcs_connector() {
rm "${temp_file}"
}
+download_aliyun_jars() {
+ local aliyun_sdk_version="3.10.2"
+ local aliyun_sdk="aliyun_java_sdk_${aliyun_sdk_version}.zip"
+ local bundle_dir="${1}"
+ local target_dir="${2}"
+ if [ ! -f "${bundle_dir}/${aliyun_sdk}" ]; then
+ curl -L -s -o "${bundle_dir}/${aliyun_sdk}"
https://gosspublic.alicdn.com/sdks/java/${aliyun_sdk}
+ fi
+ rm -rf "${bundle_dir}/aliyun"
+ unzip -q "${bundle_dir}/${aliyun_sdk}" -d "${bundle_dir}/aliyun"
+ cp
"${bundle_dir}/aliyun/aliyun_java_sdk_${aliyun_sdk_version}/aliyun-sdk-oss-3.10.2.jar"
${target_dir}
+ cp
"${bundle_dir}/aliyun/aliyun_java_sdk_${aliyun_sdk_version}/lib/hamcrest-core-1.1.jar"
${target_dir}
+ cp
"${bundle_dir}/aliyun/aliyun_java_sdk_${aliyun_sdk_version}/lib/jdom2-2.0.6.jar"
${target_dir}
+}
+
# Build the Gravitino project
${gravitino_home}/gradlew clean build -x test
@@ -93,6 +108,8 @@
iceberg_azure_bundle="iceberg-azure-bundle-${iceberg_version}.jar"
wget
"https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${iceberg_version}/${iceberg_azure_bundle}";
-O "${gravitino_staging_dir}/${iceberg_azure_bundle}"
cp "${gravitino_staging_dir}/${iceberg_azure_bundle}"
"${gravitino_iceberg_rest_dir}"
+download_aliyun_jars "${gravitino_staging_dir}" "${gravitino_iceberg_rest_dir}"
+
echo "Finish downloading"
mkdir -p "${gravitino_dir}/packages/gravitino/bin"
@@ -110,10 +127,14 @@ find ${gravitino_home}/bundles/azure-bundle/build/libs/
-name 'gravitino-azure-*
find ${gravitino_home}/bundles/gcp/build/libs/ -name 'gravitino-gcp-*.jar' !
-name '*-empty.jar' -exec cp -v {} "${gravitino_iceberg_rest_dir}" \;
find ${gravitino_home}/bundles/aws/build/libs/ -name 'gravitino-aws-*.jar' !
-name '*-empty.jar' -exec cp -v {} "${gravitino_iceberg_rest_dir}" \;
find ${gravitino_home}/bundles/azure/build/libs/ -name 'gravitino-azure-*.jar'
! -name '*-empty.jar' -exec cp -v {} "${gravitino_iceberg_rest_dir}" \;
-find ${gravitino_home}/bundles/aliyun-bundle/build/libs/ -name
'gravitino-aliyun-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${gravitino_iceberg_rest_dir}" \;
+find ${gravitino_home}/bundles/aliyun/build/libs/ -name
'gravitino-aliyun-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${gravitino_iceberg_rest_dir}" \;
download_gcs_connector
+# Temporary rm log4j from Gravition to prevent class conflict with Iceberg AWS
bundle jar
+rm -f ${gravitino_iceberg_rest_dir}/log4j-api-*.jar
+rm -f ${gravitino_iceberg_rest_dir}/log4j-core-*.jar
+
# Keeping the container running at all times
cat <<EOF >> "${gravitino_dir}/packages/gravitino/bin/gravitino.sh"
diff --git a/dev/docker/gravitino/start-gravitino.sh
b/dev/docker/gravitino/start-gravitino.sh
index 1cc612348c..608a9f7395 100755
--- a/dev/docker/gravitino/start-gravitino.sh
+++ b/dev/docker/gravitino/start-gravitino.sh
@@ -26,4 +26,7 @@ cd ${gravitino_dir}
python bin/rewrite_gravitino_server_config.py
-./bin/gravitino.sh start
\ No newline at end of file
+JAVA_OPTS+=" -XX:-UseContainerSupport"
+export JAVA_OPTS
+
+./bin/gravitino.sh start
diff --git a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
index 46af507705..9ddc1b435f 100755
--- a/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
+++ b/dev/docker/iceberg-rest-server/iceberg-rest-server-dependency.sh
@@ -22,6 +22,21 @@ iceberg_rest_server_dir="$(dirname "${BASH_SOURCE-$0}")"
iceberg_rest_server_dir="$(cd "${iceberg_rest_server_dir}">/dev/null; pwd)"
gravitino_home="$(cd "${iceberg_rest_server_dir}/../../..">/dev/null; pwd)"
+download_aliyun_jars() {
+ local aliyun_sdk_version="3.10.2"
+ local aliyun_sdk="aliyun_java_sdk_${aliyun_sdk_version}.zip"
+ local bundle_dir="${1}"
+ local target_dir="${2}"
+ if [ ! -f "${bundle_dir}/${aliyun_sdk}" ]; then
+ curl -L -s -o "${bundle_dir}/${aliyun_sdk}"
https://gosspublic.alicdn.com/sdks/java/${aliyun_sdk}
+ fi
+ rm -rf "${bundle_dir}/aliyun"
+ unzip -q "${bundle_dir}/${aliyun_sdk}" -d "${bundle_dir}/aliyun"
+ cp
"${bundle_dir}/aliyun/aliyun_java_sdk_${aliyun_sdk_version}/aliyun-sdk-oss-3.10.2.jar"
${target_dir}
+ cp
"${bundle_dir}/aliyun/aliyun_java_sdk_${aliyun_sdk_version}/lib/hamcrest-core-1.1.jar"
${target_dir}
+ cp
"${bundle_dir}/aliyun/aliyun_java_sdk_${aliyun_sdk_version}/lib/jdom2-2.0.6.jar"
${target_dir}
+}
+
# Prepare the Iceberg REST server packages
cd ${gravitino_home}
./gradlew clean assembleIcebergRESTServer -x test
@@ -38,8 +53,7 @@ cd ${gravitino_home}
./gradlew :bundles:gcp:jar
./gradlew :bundles:aws:jar
./gradlew :bundles:azure:jar
-## Iceberg doesn't provide Iceberg Aliyun bundle jar, so use Gravitino aliyun
bundle to provide OSS packages.
-./gradlew :bundles:aliyun-bundle:jar
+./gradlew :bundles:aliyun:jar
# prepare bundle jar
cd ${iceberg_rest_server_dir}
@@ -47,7 +61,7 @@ mkdir -p bundles
find ${gravitino_home}/bundles/gcp/build/libs/ -name 'gravitino-gcp-*.jar' !
-name '*-empty.jar' -exec cp -v {} bundles/ \;
find ${gravitino_home}/bundles/aws/build/libs/ -name 'gravitino-aws-*.jar' !
-name '*-empty.jar' -exec cp -v {} bundles/ \;
find ${gravitino_home}/bundles/azure/build/libs/ -name 'gravitino-azure-*.jar'
! -name '*-empty.jar' -exec cp -v {} bundles/ \;
-find ${gravitino_home}/bundles/aliyun-bundle/build/libs/ -name
'gravitino-aliyun-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {} bundles/ \;
+find ${gravitino_home}/bundles/aliyun/build/libs/ -name
'gravitino-aliyun-*.jar' ! -name '*-empty.jar' -exec cp -v {} bundles/ \;
iceberg_version="1.9.2"
@@ -66,10 +80,18 @@ if [ ! -f "bundles/${iceberg_azure_bundle}" ]; then
curl -L -s -o bundles/${iceberg_azure_bundle}
https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-azure-bundle/${iceberg_version}/${iceberg_azure_bundle}
fi
+download_aliyun_jars "bundles"
"${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/libs/"
+
# download jdbc driver
-curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar
https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar
+if [ ! -f "bundles/sqlite-jdbc-3.42.0.0.jar" ]; then
+ curl -L -s -o bundles/sqlite-jdbc-3.42.0.0.jar
https://repo1.maven.org/maven2/org/xerial/sqlite-jdbc/3.42.0.0/sqlite-jdbc-3.42.0.0.jar
+fi
cp bundles/*jar
${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/libs/
+# Temporary rm log4j from Gravition to prevent class conflict with Iceberg AWS
bundle jar
+rm -f
${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/libs/log4j-api-*.jar
+rm -f
${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/libs/log4j-core-*.jar
+
cp start-iceberg-rest-server.sh
${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/bin/
cp rewrite_config.py
${iceberg_rest_server_dir}/packages/gravitino-iceberg-rest-server/bin/
diff --git a/dev/docker/iceberg-rest-server/start-iceberg-rest-server.sh
b/dev/docker/iceberg-rest-server/start-iceberg-rest-server.sh
index ab52f8088b..6ee6152dc3 100755
--- a/dev/docker/iceberg-rest-server/start-iceberg-rest-server.sh
+++ b/dev/docker/iceberg-rest-server/start-iceberg-rest-server.sh
@@ -26,4 +26,7 @@ cd ${iceberg_rest_server_dir}
python bin/rewrite_config.py
+JAVA_OPTS+=" -XX:-UseContainerSupport"
+export JAVA_OPTS
+
exec ./bin/gravitino-iceberg-rest-server.sh run
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index c6fe455436..206195f51b 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -299,6 +299,10 @@ For other Iceberg s3 properties not managed by Gravitino
like `s3.sse.type`, you
Please refer to [S3
credentials](./security/credential-vending.md#s3-credentials) for credential
related configurations.
+:::caution
+To resolve Log4j class conflict issues that may arise when using Iceberg AWS
1.9 bundle jars alongside the Gravitino Iceberg REST server, it is recommended
to remove the Log4j JAR files (specifically log4j-core and log4j-api) from the
`iceberg-rest-server/libs` directory.
+:::
+
:::info
To configure the JDBC catalog backend, set the
`gravitino.iceberg-rest.warehouse` parameter to
`s3://{bucket_name}/${prefix_name}`. For the Hive catalog backend, set
`gravitino.iceberg-rest.warehouse` to `s3a://{bucket_name}/${prefix_name}`.
Additionally, download the [Iceberg AWS
bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-aws-bundle)
and place it in the classpath of Iceberg REST server.
:::
diff --git a/docs/lakehouse-iceberg-catalog.md
b/docs/lakehouse-iceberg-catalog.md
index 7138a49e78..bb92e38c3f 100644
--- a/docs/lakehouse-iceberg-catalog.md
+++ b/docs/lakehouse-iceberg-catalog.md
@@ -82,6 +82,10 @@ Supports using static access-key-id and secret-access-key to
access S3 data.
For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`,
you could config it directly by `gravitino.bypass.s3.sse.type`.
+:::caution
+To resolve Log4j class conflict issues that may arise when using Iceberg AWS
1.9 bundle jars alongside the Gravitino server, it is recommended to remove the
Log4j JAR files (specifically log4j-core and log4j-api) from the
`catalogs/lakehouse-iceberg/libs` directory.
+:::
+
:::info
To configure the JDBC catalog backend, set the `warehouse` parameter to
`s3://{bucket_name}/${prefix_name}`. For the Hive catalog backend, set
`warehouse` to `s3a://{bucket_name}/${prefix_name}`. Additionally, download the
[Iceberg AWS
bundle](https://mvnrepository.com/artifact/org.apache.iceberg/iceberg-aws-bundle)
and place it in the `catalogs/lakehouse-iceberg/libs/` directory.
:::
