This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new a2967b60842 [fix](lance) Fix Lance JNI compatibility with glibc 2.17
(#67862)
a2967b60842 is described below
commit a2967b60842a6c4379ea9e153bde7d313ae8871b
Author: zhangstar333 <[email protected]>
AuthorDate: Sun Sep 13 18:50:17 2026 +0800
[fix](lance) Fix Lance JNI compatibility with glibc 2.17 (#67862)
### What problem does this PR solve?
Problem Summary:
Upgrade Lance Java to 11.0.0 and automatically replace its Linux x86_64
JNI library with the glibc 2.17 build from apache/doris-thirdparty
during FE packaging.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
build.sh | 2 +-
docker/thirdparties/lance-jni-helpers.sh | 117 +++++++++++++++++++++++++++++++
docs/lance-jni-packaging.md | 42 +++++++++++
fe/pom.xml | 2 +-
post-build.sh | 10 ++-
5 files changed, 169 insertions(+), 4 deletions(-)
diff --git a/build.sh b/build.sh
index 199cf09729e..b20175e0c0d 100755
--- a/build.sh
+++ b/build.sh
@@ -1000,7 +1000,7 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
#cp -r -p "${DORIS_HOME}/docs/build/help-resource.zip"
"${DORIS_OUTPUT}/fe/lib"/
- # Third-party filesystem jars (JuiceFS, JindoFS) are packaged by
post-build.sh
+ # Third-party filesystem JARs and the Lance JNI library are packaged by
post-build.sh
"${DORIS_HOME}/post-build.sh" --fe --output "${DORIS_OUTPUT}"
cp -r -p "${DORIS_HOME}/webroot/static" "${DORIS_OUTPUT}/fe/webroot"/
diff --git a/docker/thirdparties/lance-jni-helpers.sh
b/docker/thirdparties/lance-jni-helpers.sh
new file mode 100644
index 00000000000..39a909b600b
--- /dev/null
+++ b/docker/thirdparties/lance-jni-helpers.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Keep these pins in sync with lance.version in fe/pom.xml.
+LANCE_JNI_VERSION="11.0.0"
+LANCE_JNI_ASSET="liblance_jni-11.0.0-linux-x86_64-glibc2.17-r1.so.gz"
+LANCE_JNI_URL="https://github.com/apache/doris-thirdparty/releases/download/lance-jni-11.0.0-glibc2.17-r1/${LANCE_JNI_ASSET}";
+LANCE_JNI_ARCHIVE_SHA256="f9dc713269632e26c06ea2ea20637ca16ec9d76772dc20db0bead11e5cd34a6b"
+LANCE_JNI_LIBRARY_SHA256="b6540edd3bdcd76b04f96a9e78fe804a2f1bffbd75eb35abe84de9091ba432ac"
+
+lance_jni_download() (
+ set -eo pipefail
+ local cache_dir="$1"
+ local archive="${cache_dir}/${LANCE_JNI_ASSET}"
+ local work_dir url digest
+ local -a urls=()
+
+ mkdir -p "${cache_dir}"
+ if [[ -f "${archive}" ]]; then
+ digest=$(sha256sum "${archive}" | awk '{print $1}')
+ if [[ "${digest}" == "${LANCE_JNI_ARCHIVE_SHA256}" ]]; then
+ printf '%s\n' "${archive}"
+ exit 0
+ fi
+ echo "Lance JNI cache checksum mismatch; downloading again:
${archive}" >&2
+ fi
+
+ work_dir=$(mktemp -d "${cache_dir}/.lance-jni-download.XXXXXX")
+ trap 'rm -rf -- "${work_dir}"' EXIT
+ if [[ -n "${REPOSITORY_URL:-}" ]]; then
+ urls+=("${REPOSITORY_URL%/}/${LANCE_JNI_ASSET}")
+ fi
+ urls+=("${LANCE_JNI_URL}")
+ for url in "${urls[@]}"; do
+ echo "Downloading Lance JNI from ${url}" >&2
+ if ! curl -fL --retry 3 --retry-delay 2 --connect-timeout 10 \
+ -o "${work_dir}/library.gz" "${url}"; then
+ continue
+ fi
+ digest=$(sha256sum "${work_dir}/library.gz" | awk '{print $1}')
+ if [[ "${digest}" != "${LANCE_JNI_ARCHIVE_SHA256}" ]]; then
+ echo "ERROR: Lance JNI archive SHA256 mismatch from ${url}:
expected ${LANCE_JNI_ARCHIVE_SHA256}, got ${digest}" >&2
+ continue
+ fi
+ # Concurrent builds only publish complete, verified cache files.
+ mv -f "${work_dir}/library.gz" "${archive}"
+ printf '%s\n' "${archive}"
+ exit 0
+ done
+ echo "ERROR: failed to download and verify ${LANCE_JNI_ASSET}" >&2
+ exit 1
+)
+
+lance_jni_replace() (
+ set -eo pipefail
+ local output_dir="$1"
+ local thirdparty_dir="$2"
+ local target_system="$3"
+ local target_arch="$4"
+ if [[ "${target_system}" != "Linux" || "${target_arch}" != "x86_64" ]];
then
+ exit 0
+ fi
+
+ local entry="nativelib/linux-x86-64/liblance_jni.so"
+ local target_jar="${output_dir}/fe/lib/lance-core-${LANCE_JNI_VERSION}.jar"
+ local archive work_dir source_hash packaged_hash
+ local -a lance_jars
+ shopt -s nullglob
+ lance_jars=("${output_dir}/fe/lib/"lance-core-*.jar)
+ if [[ ${#lance_jars[@]} -ne 1 || "${lance_jars[0]}" != "${target_jar}" ]];
then
+ echo "ERROR: expected exactly one lance-core-${LANCE_JNI_VERSION}.jar
in ${output_dir}/fe/lib" >&2
+ exit 1
+ fi
+ if ! unzip -Z1 "${target_jar}" | grep -Fx "${entry}" >/dev/null; then
+ echo "ERROR: missing JNI entry ${entry} in ${target_jar}" >&2
+ exit 1
+ fi
+
+ archive=$(lance_jni_download "${thirdparty_dir}/installed/lance-jni")
+ work_dir=$(mktemp -d "${output_dir}/fe/lib/.lance-jni.XXXXXX")
+ trap 'rm -rf -- "${work_dir}"' EXIT
+ mkdir -p "${work_dir}/$(dirname "${entry}")"
+ gzip -dc "${archive}" > "${work_dir}/${entry}"
+ source_hash=$(sha256sum "${work_dir}/${entry}" | awk '{print $1}')
+ if [[ "${source_hash}" != "${LANCE_JNI_LIBRARY_SHA256}" ]]; then
+ echo "ERROR: Lance JNI library SHA256 mismatch: expected
${LANCE_JNI_LIBRARY_SHA256}, got ${source_hash}" >&2
+ exit 1
+ fi
+ # Preserve the Maven cache and only replace the output JAR after
verification.
+ cp -p "${target_jar}" "${work_dir}/lance-core.jar"
+ (
+ cd "${work_dir}"
+ zip -q lance-core.jar "${entry}"
+ )
+ packaged_hash=$(unzip -p "${work_dir}/lance-core.jar" "${entry}" |
sha256sum | awk '{print $1}')
+ if [[ "${source_hash}" != "${packaged_hash}" ]]; then
+ echo "ERROR: JNI checksum mismatch in ${target_jar}" >&2
+ exit 1
+ fi
+ mv -f "${work_dir}/lance-core.jar" "${target_jar}"
+ echo "Replaced ${target_jar}!/${entry} (SHA256: ${packaged_hash})"
+)
diff --git a/docs/lance-jni-packaging.md b/docs/lance-jni-packaging.md
new file mode 100644
index 00000000000..b6941bdff2e
--- /dev/null
+++ b/docs/lance-jni-packaging.md
@@ -0,0 +1,42 @@
+# Lance JNI packaging for Linux x86_64
+
+`bash build.sh --fe` automatically replaces the Linux x86_64 JNI entry in
+`lance-core-11.0.0.jar` with the glibc 2.17 rebuild. No local Lance checkout,
+Rust toolchain, or `LANCE_JNI_SO` variable is required on the Doris build host.
+
+The artifact is pinned in `docker/thirdparties/lance-jni-helpers.sh` and
published
+in
[apache/doris-thirdparty](https://github.com/apache/doris-thirdparty/releases/tag/lance-jni-11.0.0-glibc2.17-r1).
+Both the compressed archive and the extracted library have fixed SHA256 values.
+
+The archive is cached under:
+
+```text
+thirdparty/installed/lance-jni/liblance_jni-11.0.0-linux-x86_64-glibc2.17-r1.so.gz
+```
+
+With a custom `DORIS_THIRDPARTY`, the cache is under that directory's
+`installed/lance-jni/`. A valid cache is reused without network access. A
corrupt
+cache is downloaded again. The existing `REPOSITORY_URL` mirror is tried first
+when configured, followed by the pinned GitHub Release URL. Downloads use curl;
+packaging also needs gzip, zip, unzip, and sha256sum.
+
+To update an existing FE output without compiling Doris:
+
+```bash
+bash post-build.sh --fe
+# Or use a custom output directory:
+bash post-build.sh --fe --output /path/to/output
+```
+
+Only the JAR entry `nativelib/linux-x86-64/liblance_jni.so` is changed. The
Maven
+cache, Java classes, and other native entries are preserved. Download,
checksum,
+or JAR version failures stop packaging and leave the original output JAR
intact.
+ARM64, macOS, and BE-only packaging do not download or replace this library.
+
+The rebuild retains Lance's Haswell CPU baseline (AVX2/FMA/F16C). Its GLIBC
symbol
+requirements were checked statically; a full CentOS 7 FE runtime test is still
+required. An already running FE must restart to load a newly packaged JNI
library.
+
+When upgrading Lance, update the release URL, filename, version, and both
hashes
+in the helper together with `lance.version` in `fe/pom.xml`. Offline packaging
+tests are available via `python3 thirdparty/test/lance-jni-packaging-test.py`.
diff --git a/fe/pom.xml b/fe/pom.xml
index d7a2e8dc2d1..4906cd98208 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -349,7 +349,7 @@ under the License.
<!-- Please modify iceberg.version and avro.version together,
you can find avro version info in iceberg mvn repository -->
<iceberg.version>1.11.0</iceberg.version>
- <lance.version>9.1.0-beta.3</lance.version>
+ <lance.version>11.0.0</lance.version>
<substrait.version>0.40.0</substrait.version>
<!-- 0.56.1 has bug that "SplitMode" in query response may not be
set-->
<maxcompute.version>0.53.2-public</maxcompute.version>
diff --git a/post-build.sh b/post-build.sh
index b69cd7f02ad..a0605359162 100755
--- a/post-build.sh
+++ b/post-build.sh
@@ -18,7 +18,7 @@
##############################################################
# Post-build script for packaging third-party filesystem JARs
-# (JuiceFS, JindoFS) into Doris FE/BE output directories.
+# (JuiceFS, JindoFS) and the Lance JNI library into Doris output directories.
#
# This script is independent of the main build and can be
# executed standalone or called from build.sh.
@@ -47,6 +47,7 @@ TARGET_ARCH="${TARGET_ARCH:-$(uname -m)}"
# --- Source helper scripts ---
. "${DORIS_HOME}/docker/thirdparties/juicefs-helpers.sh"
. "${DORIS_HOME}/docker/thirdparties/jindofs-helpers.sh"
+. "${DORIS_HOME}/docker/thirdparties/lance-jni-helpers.sh"
# --- JuiceFS wrapper functions (migrated from build.sh) ---
@@ -133,7 +134,8 @@ usage() {
Usage: $0 [--fe] [--be] [--output <dir>] [--juicefs] [--jindofs]
Package third-party filesystem JARs (JuiceFS, JindoFS) into Doris output
directories.
-By default, no third-party JARs are packaged. Use --juicefs/--jindofs or
environment variables to opt in.
+Linux x86_64 FE packaging automatically installs the glibc 2.17 Lance JNI
library.
+Use --juicefs/--jindofs or environment variables to opt into filesystem JAR
packaging.
Options:
--fe Process FE output directory
@@ -222,6 +224,10 @@ for target_type in fe be; do
continue
fi
+ if [[ "${target_type}" == "fe" ]]; then
+ lance_jni_replace "${OUTPUT_DIR}" "${DORIS_THIRDPARTY}"
"${TARGET_SYSTEM}" "${TARGET_ARCH}"
+ fi
+
if [[ "${BUILD_JINDOFS}" == "ON" ]]; then
install_jindofs "${target_type}" "${OUTPUT_DIR}"
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
