This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 2ce1e94b2a [#9930] improvement(build): Use soft links for Iceberg
bundle and jdbc driver jars to avoid duplication (#10377)
2ce1e94b2a is described below
commit 2ce1e94b2afaad96d84d187b11e97f7836c754c7
Author: qqqttt123 <[email protected]>
AuthorDate: Thu Mar 19 14:39:28 2026 +0800
[#9930] improvement(build): Use soft links for Iceberg bundle and jdbc
driver jars to avoid duplication (#10377)
### What changes were proposed in this pull request?
This PR refactors the Docker build process for Gravitino to avoid
duplicating Iceberg bundle and jdbc driver jars.
### Why are the changes needed?
Fix: #9930
### Does this PR introduce _any_ user-facing change?
No. This is an internal Docker build optimization that doesn't affect
user-facing functionality.
### How was this patch tested?
Manual testing of Docker build and runtime behavior.
---
dev/docker/gravitino/gravitino-dependency.sh | 34 ++++++++++++----------------
dev/docker/gravitino/start-gravitino.sh | 20 ++++++++++++++++
2 files changed, 34 insertions(+), 20 deletions(-)
diff --git a/dev/docker/gravitino/gravitino-dependency.sh
b/dev/docker/gravitino/gravitino-dependency.sh
index 824a055fe8..2994eb3283 100755
--- a/dev/docker/gravitino/gravitino-dependency.sh
+++ b/dev/docker/gravitino/gravitino-dependency.sh
@@ -22,7 +22,6 @@ gravitino_dir="$(dirname "${BASH_SOURCE-$0}")"
gravitino_dir="$(cd "${gravitino_dir}">/dev/null; pwd)"
gravitino_home="$(cd "${gravitino_dir}/../../..">/dev/null; pwd)"
gravitino_package_dir="${gravitino_dir}/packages/gravitino"
-gravitino_staging_dir="${gravitino_package_dir}/staging"
gravitino_iceberg_rest_dir="${gravitino_package_dir}/iceberg-rest-server/libs/"
# Function to download and distribute GCS connector
@@ -60,26 +59,17 @@ rm -rf "${gravitino_dir}/packages"
mkdir -p "${gravitino_dir}/packages"
cp -r "${gravitino_home}/distribution/package" "${gravitino_package_dir}"
-mkdir -p "${gravitino_staging_dir}"
echo "Start to download the jar package"
-mysql_driver="mysql-connector-java-8.0.27.jar"
-wget
"https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/$mysql_driver";
-O "${gravitino_staging_dir}/${mysql_driver}"
-cp "${gravitino_staging_dir}/${mysql_driver}"
"${gravitino_package_dir}/catalogs/jdbc-mysql/libs/"
-cp "${gravitino_staging_dir}/${mysql_driver}"
"${gravitino_package_dir}/catalogs/jdbc-doris/libs/"
-cp "${gravitino_staging_dir}/${mysql_driver}"
"${gravitino_package_dir}/catalogs/jdbc-starrocks/libs/"
-cp "${gravitino_staging_dir}/${mysql_driver}"
"${gravitino_package_dir}/catalogs/lakehouse-iceberg/libs/"
-cp "${gravitino_staging_dir}/${mysql_driver}" "${gravitino_iceberg_rest_dir}"
-cp "${gravitino_staging_dir}/${mysql_driver}" "${gravitino_package_dir}/libs/"
+jdbc_driver_dir="${gravitino_dir}/packages/gravitino/jdbc-drivers"
+mkdir -p "${jdbc_driver_dir}"
+mysql_driver="mysql-connector-java-8.0.27.jar"
+wget
"https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.27/$mysql_driver";
-O "${jdbc_driver_dir}/${mysql_driver}"
pg_driver="postgresql-42.7.0.jar"
-wget "https://jdbc.postgresql.org/download/${pg_driver}"; -O
"${gravitino_staging_dir}/${pg_driver}"
-cp "${gravitino_staging_dir}/${pg_driver}"
"${gravitino_package_dir}/catalogs/jdbc-postgresql/libs/"
-cp "${gravitino_staging_dir}/${pg_driver}"
"${gravitino_package_dir}/catalogs/lakehouse-iceberg/libs/"
-cp "${gravitino_staging_dir}/${pg_driver}" "${gravitino_iceberg_rest_dir}"
-cp "${gravitino_staging_dir}/${pg_driver}" "${gravitino_package_dir}/libs/"
+wget "https://jdbc.postgresql.org/download/${pg_driver}"; -O
"${jdbc_driver_dir}/${pg_driver}"
echo "Finish downloading"
@@ -95,11 +85,15 @@ find ${gravitino_home}/bundles/aws-bundle/build/libs/ -name
'gravitino-aws-bundl
find ${gravitino_home}/bundles/gcp-bundle/build/libs/ -name
'gravitino-gcp-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${fileset_lib_dir}" \;
find ${gravitino_home}/bundles/azure-bundle/build/libs/ -name
'gravitino-azure-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${fileset_lib_dir}" \;
-# Copy the Aliyun, AWS, GCP and Azure bundles to the Iceberg REST server libs
-find ${gravitino_home}/bundles/iceberg-gcp-bundle/build/libs/ -name
'gravitino-iceberg-gcp-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${gravitino_iceberg_rest_dir}" \;
-find ${gravitino_home}/bundles/iceberg-aws-bundle/build/libs/ -name
'gravitino-iceberg-aws-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${gravitino_iceberg_rest_dir}" \;
-find ${gravitino_home}/bundles/iceberg-azure-bundle/build/libs/ -name
'gravitino-iceberg-azure-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${gravitino_iceberg_rest_dir}" \;
-find ${gravitino_home}/bundles/iceberg-aliyun-bundle/build/libs/ -name
'gravitino-iceberg-aliyun-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${gravitino_iceberg_rest_dir}" \;
+iceberg_bundle_dir="${gravitino_package_dir}/iceberg-bundles"
+
+# Copy the Aliyun, AWS, GCP and Azure bundles to the Iceberg bundles directory
+mkdir -p "${iceberg_bundle_dir}"
+find ${gravitino_home}/bundles/iceberg-gcp-bundle/build/libs/ -name
'gravitino-iceberg-gcp-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${iceberg_bundle_dir}" \;
+find ${gravitino_home}/bundles/iceberg-aws-bundle/build/libs/ -name
'gravitino-iceberg-aws-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${iceberg_bundle_dir}" \;
+find ${gravitino_home}/bundles/iceberg-azure-bundle/build/libs/ -name
'gravitino-iceberg-azure-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${iceberg_bundle_dir}" \;
+find ${gravitino_home}/bundles/iceberg-aliyun-bundle/build/libs/ -name
'gravitino-iceberg-aliyun-bundle-*.jar' ! -name '*-empty.jar' -exec cp -v {}
"${iceberg_bundle_dir}" \;
+
download_gcs_connector
diff --git a/dev/docker/gravitino/start-gravitino.sh
b/dev/docker/gravitino/start-gravitino.sh
index 608a9f7395..58c07d3790 100755
--- a/dev/docker/gravitino/start-gravitino.sh
+++ b/dev/docker/gravitino/start-gravitino.sh
@@ -26,6 +26,26 @@ cd ${gravitino_dir}
python bin/rewrite_gravitino_server_config.py
+# Create soft links for JDBC drivers
+jdbc_driver_dir="${gravitino_dir}/jdbc-drivers"
+
+if [ -d "${jdbc_driver_dir}" ]; then
+ mkdir -p "${gravitino_dir}/libs"
+ find "${jdbc_driver_dir}" -name "mysql-connector-java-*.jar" -exec ln -sfv
{} "${gravitino_dir}/libs/" \;
+ find "${jdbc_driver_dir}" -name "postgresql-*.jar" -exec ln -sfv {}
"${gravitino_dir}/libs/" \;
+fi
+
+# Create soft links for Iceberg bundle jars
+iceberg_bundle_dir="${gravitino_dir}/iceberg-bundles"
+lakehouse_iceberg_lib_dir="${gravitino_dir}/catalogs/lakehouse-iceberg/libs"
+iceberg_rest_lib_dir="${gravitino_dir}/iceberg-rest-server/libs"
+
+if [ -d "${iceberg_bundle_dir}" ]; then
+ mkdir -p "${lakehouse_iceberg_lib_dir}"
+ mkdir -p "${iceberg_rest_lib_dir}"
+ find "${iceberg_bundle_dir}" -name '*.jar' -exec ln -sfv {}
"${lakehouse_iceberg_lib_dir}" \; -exec ln -sfv {} "${iceberg_rest_lib_dir}" \;
+fi
+
JAVA_OPTS+=" -XX:-UseContainerSupport"
export JAVA_OPTS
