This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 36f1e09531d [fix](build) Recover interrupted Paimon header and archive
installation (#68633)
36f1e09531d is described below
commit 36f1e09531d38c0884f7692b5b0347cf9e7c1ce7
Author: Gabriel <[email protected]>
AuthorDate: Wed Sep 30 21:06:57 2026 +0800
[fix](build) Recover interrupted Paimon header and archive installation
(#68633)
### What problem does this PR solve?
Related PR: #66227, #68562
An interrupted Paimon dependency installation can leave a truncated
header or a header/archive pair from different builds that the next
build incorrectly reuses.
Stage and validate the header before publishing either file, keep an
incomplete-install marker until both publications succeed, and rebuild
when that marker or an empty installed file is found. Tests inject
header-copy and both rename failures for first installs and
replacements, and verify successful repair on retry.
### Release note
Fix reuse of incomplete Paimon dependency installations.
### Check List (For Author)
- Test
- [x] Unit Test: `python3 thirdparty/test/paimon-build-test.py` (16
tests passed)
- [x] Manual test: `bash -n build.sh thirdparty/build-thirdparty.sh`
- Behavior changed:
- [x] Yes. Empty or interrupted Paimon installations trigger the
existing full third-party rebuild.
- Does this need documentation?
- [x] No.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
build.sh | 12 ++++-
thirdparty/build-thirdparty.sh | 29 ++++++++++--
thirdparty/test/paimon-build-test.py | 91 +++++++++++++++++++++++++++++++++---
3 files changed, 119 insertions(+), 13 deletions(-)
diff --git a/build.sh b/build.sh
index 9a39118c615..de304892669 100755
--- a/build.sh
+++ b/build.sh
@@ -488,10 +488,20 @@ fi
# Old installs can contain the sentinel but predate Paimon. Rebuild the whole
# dependency set so Lance and Paimon also use the same selected Rust toolchain.
+# An interrupted header/archive publication must be repaired before either
file is reused.
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
! -f "${DORIS_THIRDPARTY}/installed/lib64/liblance_c.a" ||
! -f "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
- ! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ]];
then
+ ! -f "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
+ ! -s "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
+ ! -s "${DORIS_THIRDPARTY}/installed/include/paimon_rust/paimon.h" ||
+ -e "${DORIS_THIRDPARTY}/installed/lib64/.paimon-installing" ]]; then
+ # Compilation images may contain only installed artifacts; never erase
them without a rebuild source.
+ if [[ ! -f "${DORIS_THIRDPARTY}/build-thirdparty.sh" ]]; then
+ echo "Third-party dependencies require a rebuild, but
build-thirdparty.sh is missing." >&2
+ echo "Refresh the compilation image or set DORIS_THIRDPARTY to a
complete third-party source tree." >&2
+ exit 1
+ fi
echo "Thirdparty libraries need to be build ..."
# need remove all installed pkgs because some lib like lz4 will throw
error if its lib alreay exists
rm -rf "${DORIS_THIRDPARTY}/installed"
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index eee801bd91f..52c2d481b20 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -2412,6 +2412,29 @@ install_rust_archive() {
)
}
+install_paimon_rust() {
+ (
+ set -e
+ local archive="$1"
+ local header="$2"
+ local destination="${TP_INSTALL_DIR}/include/paimon_rust/paimon.h"
+ local incomplete="${TP_INSTALL_DIR}/lib64/.paimon-installing"
+ mkdir -p "${TP_INSTALL_DIR}/include/paimon_rust"
"${TP_INSTALL_DIR}/lib64"
+ local staged
+ staged="$(mktemp "${destination}.tmp.XXXXXX")"
+ trap 'rm -f "${staged}"' EXIT
+ cp -p "${header}" "${staged}"
+ test -s "${staged}"
+ test -s "${archive}"
+ # The two renames cannot be atomic together. Keep this recovery marker
until
+ # both succeed so build.sh never reuses a mixed pair after
interruption.
+ touch "${incomplete}"
+ install_rust_archive "${archive}"
+ mv -f "${staged}" "${destination}"
+ rm -f "${incomplete}"
+ )
+}
+
build_lance_c() {
check_if_source_exist "${LANCE_C_SOURCE}"
cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}"
@@ -2607,11 +2630,7 @@ EOF
--config "${cbindgen_toml}" \
--output "${BUILD_DIR}/release/paimon.h"
- mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
- rm -rf "${TP_INSTALL_DIR}/include/paimon_rust"
- mkdir -p "${TP_INSTALL_DIR}/include/paimon_rust"
- cp -v "${BUILD_DIR}/release/paimon.h"
"${TP_INSTALL_DIR}/include/paimon_rust/"
- install_rust_archive "${BUILD_DIR}/release/libpaimon_c.a"
+ install_paimon_rust "${BUILD_DIR}/release/libpaimon_c.a"
"${BUILD_DIR}/release/paimon.h"
}
if [[ "${#packages[@]}" -eq 0 ]]; then
diff --git a/thirdparty/test/paimon-build-test.py
b/thirdparty/test/paimon-build-test.py
index 3fef1ffe6fc..c9443166453 100644
--- a/thirdparty/test/paimon-build-test.py
+++ b/thirdparty/test/paimon-build-test.py
@@ -53,7 +53,7 @@ class PaimonBuildTest(unittest.TestCase):
return subprocess.run(["bash", "-ec", script], env=self.env,
cwd=self.work, text=True, capture_output=True)
- def check_install(self, system, missing, clean=0):
+ def check_install(self, system, missing, clean=0, empty=None,
expect_rebuild=False, installed_only=False):
script = (ROOT / "build.sh").read_text()
gate = script[script.index("# build thirdparty libraries if
necessary."):
script.index("update_submodule() {")]
@@ -64,15 +64,21 @@ class PaimonBuildTest(unittest.TestCase):
"include/paimon_rust/paimon.h"]
for name in files:
if name != missing:
- self.write("installed/" + name)
+ self.write("installed/" + name, "" if name == empty else
"complete")
# Execute the real installation gate, replacing only the expensive
build.
- self.executable("build-thirdparty.sh", 'printf "%s\\n" "$*" >
"$BUILD_LOG"\n')
+ if not installed_only:
+ self.executable("build-thirdparty.sh", 'printf "%s\\n" "$*" >
"$BUILD_LOG"\n')
self.env.update(DORIS_THIRDPARTY=str(self.work), TARGET_SYSTEM=system,
CLEAN=str(clean), PARALLEL="2",
BUILD_LOG=str(self.work / "build.log"))
result = self.run_bash(gate)
+ if installed_only and (missing is not None or expect_rebuild):
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn("Refresh the compilation image", result.stderr)
+ self.assertEqual((self.work /
"installed/lib64/liblance_c.a").read_text(), "complete")
+ return
self.assertEqual(result.returncode, 0, result.stderr)
- self.assertEqual((self.work / "build.log").exists(), missing is not
None)
- if missing is not None:
+ self.assertEqual((self.work / "build.log").exists(), missing is not
None or expect_rebuild)
+ if missing is not None or expect_rebuild:
args = (self.work / "build.log").read_text().strip()
self.assertEqual(args, "-j 2" + (" --clean" if clean else ""))
# A full rebuild also replaces Lance built with the previous Rust
version.
@@ -96,9 +102,80 @@ class PaimonBuildTest(unittest.TestCase):
def test_complete_install_is_reused(self):
self.check_install("Linux", None)
+ def test_installed_only_image_preserves_dependencies(self):
+ self.check_install("Linux", "lib64/libpaimon_c.a", installed_only=True)
+
+ def test_installed_only_complete_image_is_reused(self):
+ self.check_install("Linux", None, installed_only=True)
+
+ def test_installed_only_interrupted_pair_preserves_dependencies(self):
+ self.write("installed/lib64/.paimon-installing")
+ self.check_install("Linux", None, expect_rebuild=True,
installed_only=True)
+
+ def test_incomplete_pair_rebuilds(self):
+ self.write("installed/lib64/.paimon-installing")
+ self.check_install("Linux", None, expect_rebuild=True)
+
+ def test_empty_header_rebuilds(self):
+ self.check_install("Linux", None,
empty="include/paimon_rust/paimon.h", expect_rebuild=True)
+
+ def test_empty_archive_rebuilds(self):
+ self.check_install("Linux", None, empty="lib64/libpaimon_c.a",
expect_rebuild=True)
+
def archive_installer(self):
script = (ROOT / "thirdparty/build-thirdparty.sh").read_text()
- return re.search(r"^install_rust_archive\(\) \{\n.*?^\}", script, re.M
| re.S)[0]
+ return "\n".join(re.search(r"^" + name + r"\(\) \{\n.*?^\}", script,
re.M | re.S)[0]
+ for name in ("install_rust_archive",
"install_paimon_rust"))
+
+ def test_header_archive_pair_publication(self):
+ archive = self.write("build/libpaimon_c.a", "new archive")
+ header = self.write("build/paimon.h", "new header")
+ target_archive = self.work / "installed/lib64/libpaimon_c.a"
+ target_header = self.work / "installed/include/paimon_rust/paimon.h"
+ marker = target_archive.parent / ".paimon-installing"
+ self.env.update(TP_INSTALL_DIR=str(self.work / "installed"),
STRIP_TP_LIB="OFF",
+ KERNEL="Linux", ARCHIVE=str(archive),
HEADER=str(header))
+ failures = {
+ "header-copy": 'cp() { if [[ "$1" = -p && "$2" = "$HEADER" ]];
then '
+ 'printf partial > "${@: -1}"; return 1; fi; command
cp "$@"; }',
+ "archive-publish": 'mv() { if [[ "${@: -1}" = */libpaimon_c.a ]];
then return 1; fi; '
+ 'command mv "$@"; }',
+ "header-publish": 'mv() { if [[ "${@: -1}" = */paimon.h ]]; then
return 1; fi; '
+ 'command mv "$@"; }',
+ "success": "",
+ }
+ for replacement in (False, True):
+ for stage, injection in failures.items():
+ with self.subTest(replacement=replacement, stage=stage):
+ for path in (target_archive, target_header, marker):
+ if path.exists():
+ path.unlink()
+ path.parent.mkdir(parents=True, exist_ok=True)
+ if replacement:
+ target_archive.write_text("old archive")
+ target_header.write_text("old header")
+ result = self.run_bash(self.archive_installer() + "\n" +
injection
+ + '\ninstall_paimon_rust "$ARCHIVE"
"$HEADER"')
+ if stage == "success":
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(target_header.read_text(), "new
header")
+ self.assertEqual(target_archive.read_text(), "new
archive")
+ self.assertFalse(marker.exists())
+ else:
+ self.assertNotEqual(result.returncode, 0)
+ if replacement:
+ self.assertEqual(target_header.read_text(), "old
header")
+ else:
+ self.assertFalse(target_header.exists())
+ self.assertEqual(marker.exists(), stage !=
"header-copy")
+ # A retry must repair the pair and clear its
incomplete marker.
+ retry = self.run_bash(self.archive_installer()
+ + '\ninstall_paimon_rust
"$ARCHIVE" "$HEADER"')
+ self.assertEqual(retry.returncode, 0, retry.stderr)
+ self.assertEqual(target_header.read_text(), "new
header")
+ self.assertEqual(target_archive.read_text(), "new
archive")
+ self.assertFalse(marker.exists())
+
self.assertEqual(list(target_header.parent.glob("*.tmp.*")), [])
def check_archive_publication(self, name):
source = self.write("build/" + name, "new archive")
@@ -150,7 +227,7 @@ class PaimonBuildTest(unittest.TestCase):
--version) echo 'cargo 1.94.0' ;;
build)
mkdir -p "$CARGO_TARGET_DIR/release"
- touch "$CARGO_TARGET_DIR/release/libpaimon_c.a"
+ printf 'archive' > "$CARGO_TARGET_DIR/release/libpaimon_c.a"
;;
metadata) printf '%s' "${CARGO_NET_OFFLINE:-unset}" > "$METADATA_LOG" ;;
*) exit 90 ;;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
