This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new dc28b8bb fix(cpp): link the cxx runtime in the file group example
(#962)
dc28b8bb is described below
commit dc28b8bbca4ad1e5df438acd7eefe27d47e3abdf
Author: slfan1989 <[email protected]>
AuthorDate: Sun Nov 8 12:59:14 2026 +0800
fix(cpp): link the cxx runtime in the file group example (#962)
Fix the C++ example's missing `rust::Error` symbols by linking the matching
cxx runtime.
Add a CTest regression test and preserve test exit codes after Docker
cleanup so failures correctly fail CI.
---
.github/workflows/ci.yml | 2 +
AGENTS.md | 1 +
Makefile | 13 ++-
cpp/CMakeLists.txt | 41 ++++++++-
demo/README.md | 14 ++++
demo/apps/hudi-file-group-api/cpp/CMakeLists.txt | 26 +++---
demo/apps/hudi-file-group-api/cpp/link_test.cpp | 53 ++++++++++++
demo/ci_run.sh | 6 +-
demo/compose.yaml | 3 +
demo/infra/runner/Dockerfile | 28 +++++--
demo/tests/test_ci_run.py | 102 +++++++++++++++++++++++
11 files changed, 264 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7a0f0dc0..1bb0951b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -287,6 +287,8 @@ jobs:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
+ - name: Test integration runner exit statuses
+ run: make test-demo-ci
- name: Set up Docker Buildx
uses:
docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Pre-pull base images
diff --git a/AGENTS.md b/AGENTS.md
index 4722e15e..6e4e2711 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -51,6 +51,7 @@ the `hudi-datafusion` test binaries and the Python extension
fail to link, see
make setup-venv && source .venv/bin/activate
make develop # build workspace + install Python binding via
maturin
make format check test # the pre-PR loop CI runs
+make test-demo-ci # integration runner regression tests; no Docker
required
cargo +1.94.1 check --workspace --all-targets --all-features # optional MSRV
sanity check
cargo test -p hudi-core # one crate
diff --git a/Makefile b/Makefile
index 6452694a..9dd123ca 100644
--- a/Makefile
+++ b/Makefile
@@ -22,6 +22,7 @@ SHELL := /bin/bash
VENV := .venv
PYTHON_DIR = python
UPSERT_BENCH_DIR := benchmark/upsert
+DEMO_TESTS_DIR := demo/tests
MATURIN_VERSION := $(shell grep 'requires =' $(PYTHON_DIR)/pyproject.toml |
cut -d= -f2- | tr -d '[ "]')
PACKAGE_VERSION := $(shell grep version Cargo.toml | head -n 1 | awk '{print
$$3}' | tr -d '"' )
@@ -102,7 +103,7 @@ format-rust: ## Format Rust code
.PHONY: format-python
format-python: ## Format Python code
$(info --- Format Python code ---)
- ruff format $(PYTHON_DIR) $(UPSERT_BENCH_DIR)
+ ruff format $(PYTHON_DIR) $(UPSERT_BENCH_DIR) $(DEMO_TESTS_DIR)
.PHONY: check
check: check-rust check-python ## Run check on Rust and Python
@@ -117,14 +118,18 @@ check-rust: ## Run check on Rust
.PHONY: check-python
check-python: ## Run check on Python
$(info --- Check Python format ---)
- ruff format --check --diff $(PYTHON_DIR) $(UPSERT_BENCH_DIR)
+ ruff format --check --diff $(PYTHON_DIR) $(UPSERT_BENCH_DIR)
$(DEMO_TESTS_DIR)
$(info --- Check Python linting ---)
- ruff check $(PYTHON_DIR) $(UPSERT_BENCH_DIR)
+ ruff check $(PYTHON_DIR) $(UPSERT_BENCH_DIR) $(DEMO_TESTS_DIR)
$(info --- Check Python typing ---)
pushd $(PYTHON_DIR); mypy .; popd
.PHONY: test
-test: test-rust test-python ## Run tests on Rust and Python
+test: test-demo-ci test-rust test-python ## Run integration runner, Rust and
Python tests
+
+.PHONY: test-demo-ci
+test-demo-ci: ## Test integration runner exit statuses without Docker
+ python3 -m unittest discover -s demo/tests -p 'test_ci_run.py' -v
.PHONY: test-rust
test-rust: ## Run tests on Rust
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index c7c41a94..badbd8db 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -23,13 +23,51 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
# Find the cargo metadata to locate the project paths
execute_process(
- COMMAND cargo metadata --format-version=1
+ COMMAND cargo metadata --format-version=1 --manifest-path
"${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml"
OUTPUT_VARIABLE CARGO_METADATA
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/..
+ COMMAND_ERROR_IS_FATAL ANY
)
string(JSON CARGO_WORKSPACE_ROOT GET ${CARGO_METADATA} "workspace_root")
string(JSON TARGET_DIRECTORY GET ${CARGO_METADATA} "target_directory")
+# Resolve this crate's cxx dependency: another workspace member may use a
different version.
+string(JSON HUDI_CPP_PACKAGE_ID GET "${CARGO_METADATA}" resolve root)
+string(JSON CARGO_NODES GET "${CARGO_METADATA}" resolve nodes)
+string(JSON NODE_COUNT LENGTH "${CARGO_NODES}")
+math(EXPR NODE_LAST "${NODE_COUNT} - 1")
+foreach(NODE_INDEX RANGE ${NODE_LAST})
+ string(JSON NODE_ID GET "${CARGO_NODES}" ${NODE_INDEX} id)
+ if(NODE_ID STREQUAL HUDI_CPP_PACKAGE_ID)
+ string(JSON CPP_DEPS GET "${CARGO_NODES}" ${NODE_INDEX} deps)
+ string(JSON DEP_COUNT LENGTH "${CPP_DEPS}")
+ math(EXPR DEP_LAST "${DEP_COUNT} - 1")
+ foreach(DEP_INDEX RANGE ${DEP_LAST})
+ string(JSON DEP_NAME GET "${CPP_DEPS}" ${DEP_INDEX} name)
+ if(DEP_NAME STREQUAL "cxx")
+ string(JSON CXX_PACKAGE_ID GET "${CPP_DEPS}" ${DEP_INDEX} pkg)
+ break()
+ endif()
+ endforeach()
+ break()
+ endif()
+endforeach()
+
+string(JSON PACKAGE_COUNT LENGTH "${CARGO_METADATA}" packages)
+math(EXPR PACKAGE_LAST "${PACKAGE_COUNT} - 1")
+foreach(PACKAGE_INDEX RANGE ${PACKAGE_LAST})
+ string(JSON PACKAGE_ID GET "${CARGO_METADATA}" packages ${PACKAGE_INDEX}
id)
+ if(PACKAGE_ID STREQUAL CXX_PACKAGE_ID)
+ string(JSON CXX_MANIFEST_PATH GET "${CARGO_METADATA}" packages
${PACKAGE_INDEX} manifest_path)
+ get_filename_component(CXX_CRATE_DIR "${CXX_MANIFEST_PATH}" DIRECTORY)
+ set(CXX_RUNTIME_SOURCE "${CXX_CRATE_DIR}/src/cxx.cc")
+ break()
+ endif()
+endforeach()
+if(NOT CXX_RUNTIME_SOURCE OR NOT EXISTS "${CXX_RUNTIME_SOURCE}")
+ message(FATAL_ERROR "Could not locate the hudi-cpp cxx dependency's C++
runtime source")
+endif()
+
# Set the default build type if not specified
if(NOT CMAKE_BUILD_TYPE)
set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Build type" FORCE)
@@ -83,6 +121,7 @@ target_include_directories(hudi INTERFACE
# Store the library path as a property for consumers
set_target_properties(hudi PROPERTIES
INTERFACE_HUDI_LIBRARY_PATH ${RUST_LIB_PATH}
+ INTERFACE_HUDI_CXX_RUNTIME_SOURCE "${CXX_RUNTIME_SOURCE}"
)
# Make sure the Rust library is built before anyone uses this target
diff --git a/demo/README.md b/demo/README.md
index 8934b402..2558b159 100644
--- a/demo/README.md
+++ b/demo/README.md
@@ -21,10 +21,19 @@
This directory contains demo applications showcasing Hudi-rs APIs across
different programming languages. The demos run on Docker Compose and also serve
as integration tests in the CI pipeline (see
[`.github/workflows/ci.yml`](../.github/workflows/ci.yml)).
+Run `make test-demo-ci` from the repository root to check the integration
runner's
+exit-status handling without Docker. These tests verify that cleanup runs after
+the demo and preserves its result, including a nonzero CTest exit status.
Cleanup
+failures remain warnings and do not replace the demo's exit status.
+
## Infrastructure
The demo infrastructure is defined in [`compose.yaml`](compose.yaml).
+The runner includes libclang and keeps uv and Python outside `/root`. Compose
+passes the host UID/GID to make `/opt/.venv` writable. Image builds check these
+dependencies and permissions as the runtime user.
+
Test data copied from the [`crates/test`](../crates/test/data) are placed in
Minio, providing COW and MOR tables at:
- `s3://hudi-demo/cow/`
- `s3://hudi-demo/mor/`
@@ -42,3 +51,8 @@ Test data copied from the
[`crates/test`](../crates/test/data) are placed in Min
### [`hudi-file-group-api/cpp`](apps/hudi-file-group-api/cpp)
**C++**: File group reading using C++ bindings and Arrow C ABI.
+
+The C++ example also registers a CTest that constructs a file slice and
catches a
+`rust::Error` from an invalid file name. It checks the generated bridge and cxx
+runtime link together without requiring MinIO or cloud credentials. After
building
+the example, run `ctest --test-dir <example-build-directory>
--output-on-failure`.
diff --git a/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
b/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
index b8ffff15..d82bf8b4 100644
--- a/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
+++ b/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt
@@ -21,6 +21,8 @@ project(hudi_cpp_example)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
+include(CTest)
+
add_subdirectory(../../../../cpp hudi-cpp)
add_executable(file_group_api_cpp
@@ -32,23 +34,25 @@ get_target_property(HUDI_LIB_PATH hudi
INTERFACE_HUDI_LIBRARY_PATH)
get_filename_component(TARGET_DIR ${HUDI_LIB_PATH} DIRECTORY)
get_filename_component(TARGET_DIR ${TARGET_DIR} DIRECTORY) # Go up one level
to get target directory
set(CXX_BRIDGE_DIR "${TARGET_DIR}/cxxbridge")
+get_target_property(CXX_RUNTIME_SOURCE hudi INTERFACE_HUDI_CXX_RUNTIME_SOURCE)
+
+set_source_files_properties(${CXX_BRIDGE_DIR}/hudi/src/lib.rs.cc PROPERTIES
GENERATED TRUE)
-target_sources(file_group_api_cpp PRIVATE
+# The Rust cdylib exports the C bridge but can hide C++ symbols such as
rust::Error.
+add_library(hudi_cpp_example_bridge STATIC
${CXX_BRIDGE_DIR}/hudi/src/lib.rs.cc
+ "${CXX_RUNTIME_SOURCE}"
)
-
-# Make sure the executable depends on the rust build completing
-add_dependencies(file_group_api_cpp rust_build)
-
-# Link against the hudi library target
-target_link_libraries(file_group_api_cpp
+target_link_libraries(hudi_cpp_example_bridge PUBLIC
Hudi::Hudi
+ "${HUDI_LIB_PATH}"
)
+target_link_libraries(file_group_api_cpp PRIVATE hudi_cpp_example_bridge)
-# Get the actual library path and link it directly
-get_target_property(HUDI_LIB_PATH hudi INTERFACE_HUDI_LIBRARY_PATH)
-if(HUDI_LIB_PATH)
- target_link_libraries(file_group_api_cpp ${HUDI_LIB_PATH})
+if(BUILD_TESTING)
+ add_executable(file_group_api_cpp_link_test link_test.cpp)
+ target_link_libraries(file_group_api_cpp_link_test PRIVATE
hudi_cpp_example_bridge)
+ add_test(NAME file_group_api_cpp_link_test COMMAND
file_group_api_cpp_link_test)
endif()
# Output configuration
diff --git a/demo/apps/hudi-file-group-api/cpp/link_test.cpp
b/demo/apps/hudi-file-group-api/cpp/link_test.cpp
new file mode 100644
index 00000000..4ec55dfa
--- /dev/null
+++ b/demo/apps/hudi-file-group-api/cpp/link_test.cpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+#include <cstdlib>
+#include <exception>
+#include <iostream>
+#include <string>
+#include <vector>
+
+#include "cxx.h"
+#include "src/lib.rs.h"
+
+int main() {
+ const std::vector<std::string> logs;
+ try {
+ const auto slice = new_file_slice_from_file_names(
+ "",
"f00000000-0000-0000-0000-000000000000-0_0-1-0_20250101000000000.parquet",
logs);
+ } catch (const std::exception& error) {
+ std::cerr << "Valid file-slice construction failed: " << error.what()
<< '\n';
+ return EXIT_FAILURE;
+ }
+
+ try {
+ const auto slice = new_file_slice_from_file_names("",
"invalid.parquet", logs);
+ } catch (const rust::Error& error) {
+ if (std::string(error.what()).empty()) {
+ std::cerr << "rust::Error did not preserve its error message\n";
+ return EXIT_FAILURE;
+ }
+ return EXIT_SUCCESS;
+ } catch (const std::exception& error) {
+ std::cerr << "Expected rust::Error, got: " << error.what() << '\n';
+ return EXIT_FAILURE;
+ }
+
+ std::cerr << "Invalid file name did not throw rust::Error\n";
+ return EXIT_FAILURE;
+}
diff --git a/demo/ci_run.sh b/demo/ci_run.sh
index 32f130d3..2333c0b2 100755
--- a/demo/ci_run.sh
+++ b/demo/ci_run.sh
@@ -76,6 +76,7 @@ elif [ "$app_path" = "hudi-file-group-api/cpp" ]; then
mkdir build && cd build && \
cmake .. && \
make && \
+ ctest --output-on-failure && \
./file_group_api_cpp
"
else
@@ -83,5 +84,8 @@ else
exit 1
fi
-# Always tear down the compose stack to release file locks and avoid cache
save issues
+integration_status=$?
+
+# Cleanup must not replace the exit status of the demo or CTest.
docker compose down -v || echo 'Warning: Failed to tear down compose stack' >&2
+exit "$integration_status"
diff --git a/demo/compose.yaml b/demo/compose.yaml
index 2d50ba74..3e8d986d 100644
--- a/demo/compose.yaml
+++ b/demo/compose.yaml
@@ -57,6 +57,9 @@ services:
runner:
build:
context: ./infra/runner
+ args:
+ RUNNER_UID: "${HOST_UID:-1000}"
+ RUNNER_GID: "${HOST_GID:-1000}"
container_name: runner
user: "${HOST_UID}:${HOST_GID}"
volumes:
diff --git a/demo/infra/runner/Dockerfile b/demo/infra/runner/Dockerfile
index d0a2637a..cd26a7ee 100644
--- a/demo/infra/runner/Dockerfile
+++ b/demo/infra/runner/Dockerfile
@@ -17,19 +17,16 @@
FROM rust:1.94
-RUN apt-get update && apt-get install -y cmake curl ca-certificates
protobuf-compiler
+RUN apt-get update && apt-get install -y cmake curl ca-certificates
protobuf-compiler clang libclang-dev
ADD https://astral.sh/uv/install.sh /uv-installer.sh
-RUN sh /uv-installer.sh && rm /uv-installer.sh
+RUN UV_UNMANAGED_INSTALL=/usr/local/bin sh /uv-installer.sh && rm
/uv-installer.sh
-ENV PATH="/root/.local/bin/:$PATH"
+# The runner uses the host UID, so Python must be accessible outside /root.
+ENV UV_PYTHON_INSTALL_DIR=/opt/python
-RUN uv python install 3.13
-
-RUN uv python list
-
-RUN uv venv /opt/.venv
+RUN uv python install --no-bin 3.13 && uv venv --python 3.13 /opt/.venv
ENV VIRTUAL_ENV=/opt/.venv
ENV PATH="/opt/.venv/bin:$PATH"
@@ -38,4 +35,19 @@ RUN uv pip install --only-binary=all 'pyarrow==20.0.0'
WORKDIR /opt
+ARG RUNNER_UID=1000
+ARG RUNNER_GID=1000
+
+# make develop installs into this environment as the host user.
+RUN chown -R "${RUNNER_UID}:${RUNNER_GID}" /opt/.venv
+
+USER ${RUNNER_UID}:${RUNNER_GID}
+
+# Fail early rather than discovering missing dependencies during integration
tests.
+RUN uv --version \
+ && python -c 'import pyarrow' \
+ && python -c 'import ctypes, glob;
ctypes.CDLL(glob.glob("/usr/lib/llvm-*/lib/libclang.so")[0])' \
+ && test -w /opt/.venv/bin \
+ && test -w /opt/.venv/lib/python3.13/site-packages
+
CMD tail -f /dev/null
diff --git a/demo/tests/test_ci_run.py b/demo/tests/test_ci_run.py
new file mode 100644
index 00000000..ce01df94
--- /dev/null
+++ b/demo/tests/test_ci_run.py
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Check integration runner exit statuses without starting Docker."""
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+class CiRunTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory(prefix="hudi ci test ")
+ self.addCleanup(self.temp.cleanup)
+ self.root = Path(self.temp.name)
+ self.log = self.root / "docker.log"
+ self.script = Path(__file__).resolve().parents[1] / "ci_run.sh"
+ docker = self.root / "docker"
+ docker.write_text(
+ """#!/bin/bash
+if [[ "$1" == inspect ]]; then
+ echo inspect >> "$DEMO_TEST_DOCKER_LOG"
+ echo running
+elif [[ "$1" == compose ]]; then
+ echo "$2" >> "$DEMO_TEST_DOCKER_LOG"
+ case "$2" in
+ exec) exit "$DEMO_TEST_EXEC_STATUS" ;;
+ down) exit "$DEMO_TEST_CLEANUP_STATUS" ;;
+ esac
+fi
+exit 0
+"""
+ )
+ docker.chmod(0o755)
+
+ def run_runner(self, exec_status, cleanup_status=0,
app="hudi-file-group-api/cpp"):
+ self.log.unlink(missing_ok=True)
+ env = dict(os.environ)
+ env.update(
+ PATH=str(self.root) + os.pathsep + env["PATH"],
+ DEMO_TEST_DOCKER_LOG=str(self.log),
+ DEMO_TEST_EXEC_STATUS=str(exec_status),
+ DEMO_TEST_CLEANUP_STATUS=str(cleanup_status),
+ )
+ result = subprocess.run(
+ ["bash", str(self.script), app],
+ cwd=self.root,
+ env=env,
+ text=True,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ check=False,
+ timeout=10,
+ )
+ self.assertEqual(
+ self.log.read_text().splitlines(), ["up", "inspect", "exec",
"down"]
+ )
+ return result
+
+ def test_success_returns_zero_and_cleans_up(self):
+ result = self.run_runner(0)
+ self.assertEqual(result.returncode, 0, result.stdout)
+
+ def test_ctest_failure_is_not_hidden_by_successful_cleanup(self):
+ result = self.run_runner(8)
+ self.assertEqual(result.returncode, 8, result.stdout)
+
+ def test_cleanup_failure_preserves_original_test_failure(self):
+ result = self.run_runner(8, cleanup_status=9)
+ self.assertEqual(result.returncode, 8, result.stdout)
+ self.assertIn("Warning: Failed to tear down compose stack",
result.stdout)
+
+ def test_cleanup_failure_after_success_remains_a_warning(self):
+ result = self.run_runner(0, cleanup_status=9)
+ self.assertEqual(result.returncode, 0, result.stdout)
+ self.assertIn("Warning: Failed to tear down compose stack",
result.stdout)
+
+ def test_other_demo_failures_are_preserved(self):
+ for app in ("datafusion", "hudi-table-api/rust",
"hudi-table-api/python"):
+ with self.subTest(app=app):
+ result = self.run_runner(37, app=app)
+ self.assertEqual(result.returncode, 37, result.stdout)
+
+
+if __name__ == "__main__":
+ unittest.main()