This is an automated email from the ASF dual-hosted git repository. xushiyan pushed a commit to branch ci-parallelize-integ-test in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
commit ff0c2915f1a1ecf1aaf95ca893ab4a081717186d Author: Shiyan Xu <[email protected]> AuthorDate: Mon Jun 2 01:57:39 2025 -0500 chore(ci): parallel run integration tests --- .../{sql-datafusion => apps/datafusion}/Cargo.toml | 2 +- .../datafusion}/src/main.rs | 0 .../hudi-file-group-api}/cpp/CMakeLists.txt | 4 +- .../hudi-file-group-api}/cpp/main.cpp | 0 .../hudi-table-api/python}/src/__init__.py | 0 .../hudi-table-api/python}/src/main.py | 0 .../hudi-table-api/rust}/Cargo.toml | 2 +- .../hudi-table-api/rust}/src/main.rs | 0 demo/ci_run.sh | 78 ++++++++++++++++++++++ demo/run_demo.sh | 56 ---------------- demo/sql-datafusion/run.sh | 21 ------ demo/table-api-python/run.sh | 21 ------ demo/table-api-rust/run.sh | 21 ------ 13 files changed, 81 insertions(+), 124 deletions(-) diff --git a/demo/sql-datafusion/Cargo.toml b/demo/apps/datafusion/Cargo.toml similarity index 93% rename from demo/sql-datafusion/Cargo.toml rename to demo/apps/datafusion/Cargo.toml index c9babe1..4938de9 100644 --- a/demo/sql-datafusion/Cargo.toml +++ b/demo/apps/datafusion/Cargo.toml @@ -26,4 +26,4 @@ edition = "2021" [dependencies] tokio = "^1" datafusion = "~45.0.0" -hudi = { path = "../../crates/hudi", features = ["datafusion"] } +hudi = { path = "../../../crates/hudi", features = ["datafusion"] } diff --git a/demo/sql-datafusion/src/main.rs b/demo/apps/datafusion/src/main.rs similarity index 100% rename from demo/sql-datafusion/src/main.rs rename to demo/apps/datafusion/src/main.rs diff --git a/demo/file-group-api/cpp/CMakeLists.txt b/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt similarity index 94% rename from demo/file-group-api/cpp/CMakeLists.txt rename to demo/apps/hudi-file-group-api/cpp/CMakeLists.txt index e1ac93f..b8ffff1 100644 --- a/demo/file-group-api/cpp/CMakeLists.txt +++ b/demo/apps/hudi-file-group-api/cpp/CMakeLists.txt @@ -21,10 +21,8 @@ project(hudi_cpp_example) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -# Add the hudi-cpp subdirectory (adjust path as needed) -add_subdirectory(../../../cpp hudi-cpp) +add_subdirectory(../../../../cpp hudi-cpp) -# Add the example executable add_executable(file_group_api_cpp main.cpp ) diff --git a/demo/file-group-api/cpp/main.cpp b/demo/apps/hudi-file-group-api/cpp/main.cpp similarity index 100% rename from demo/file-group-api/cpp/main.cpp rename to demo/apps/hudi-file-group-api/cpp/main.cpp diff --git a/demo/table-api-python/src/__init__.py b/demo/apps/hudi-table-api/python/src/__init__.py similarity index 100% rename from demo/table-api-python/src/__init__.py rename to demo/apps/hudi-table-api/python/src/__init__.py diff --git a/demo/table-api-python/src/main.py b/demo/apps/hudi-table-api/python/src/main.py similarity index 100% rename from demo/table-api-python/src/main.py rename to demo/apps/hudi-table-api/python/src/main.py diff --git a/demo/table-api-rust/Cargo.toml b/demo/apps/hudi-table-api/rust/Cargo.toml similarity index 95% rename from demo/table-api-rust/Cargo.toml rename to demo/apps/hudi-table-api/rust/Cargo.toml index c807db6..39139a2 100644 --- a/demo/table-api-rust/Cargo.toml +++ b/demo/apps/hudi-table-api/rust/Cargo.toml @@ -27,4 +27,4 @@ edition = "2021" tokio = "^1" arrow = { version = "~54.1.0", features = ["pyarrow"] } -hudi = { path = "../../crates/hudi" } +hudi = { path = "../../../../crates/hudi" } diff --git a/demo/table-api-rust/src/main.rs b/demo/apps/hudi-table-api/rust/src/main.rs similarity index 100% rename from demo/table-api-rust/src/main.rs rename to demo/apps/hudi-table-api/rust/src/main.rs diff --git a/demo/ci_run.sh b/demo/ci_run.sh new file mode 100755 index 0000000..a27f748 --- /dev/null +++ b/demo/ci_run.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +docker compose up --build -d + +max_attempts=30 +attempt=0 + +until [ "$(docker inspect -f '{{.State.Status}}' runner)" = "running" ] || [ $attempt -eq $max_attempts ]; do + attempt=$(( $attempt + 1 )) + echo "Waiting for container... (attempt $attempt of $max_attempts)" + sleep 1 +done + +if [ $attempt -eq $max_attempts ]; then + echo "Container failed to become ready in time" + exit 1 +fi + +app_path=$1 +if [ -z "$app_path" ]; then + echo "Usage: $0 <path_to_app>" + exit 1 +fi + +app_path_in_container="/opt/hudi-rs/demo/$app_path" +if [ "$app_path" = "datafusion" ]; then + docker compose exec -T runner /bin/bash -c " + source /opt/venv/bin/activate && \ + cd $app_path_in_container && \ + make setup develop && \ + cargo run -- --no-build --no-tests + " +elif [ "$app_path" = "hudi-table-api/rust" ]; then + docker compose exec -T runner /bin/bash -c " + source /opt/venv/bin/activate && \ + cd $app_path_in_container && \ + make setup develop && \ + cargo run -- --no-build --no-tests + " +elif [ "$app_path" = "hudi-table-api/python" ]; then + docker compose exec -T runner /bin/bash -c " + source /opt/venv/bin/activate && \ + cd $app_path_in_container && \ + make setup develop && \ + cargo run -- --no-build --no-tests + " +elif [ "$app_path" = "hudi-file-group-api/cpp" ]; then + docker compose exec -T runner /bin/bash -c " + cd /opt/hudi-rs/cpp && \ + cargo build --release && \ + cd $app_path_in_container && \ + mkdir build && cd build && \ + cmake .. && \ + make && \ + ./file_group_api_cpp + " +else + echo "Unknown app path: $app_path" + exit 1 +fi diff --git a/demo/run_demo.sh b/demo/run_demo.sh deleted file mode 100755 index 32d9330..0000000 --- a/demo/run_demo.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -docker compose up --build -d - -max_attempts=30 -attempt=0 - -until [ "$(docker inspect -f '{{.State.Status}}' runner)" = "running" ] || [ $attempt -eq $max_attempts ]; do - attempt=$(( $attempt + 1 )) - echo "Waiting for container... (attempt $attempt of $max_attempts)" - sleep 1 -done - -if [ $attempt -eq $max_attempts ]; then - echo "Container failed to become ready in time" - exit 1 -fi - -# Run the C++ demo app -docker compose exec -T runner /bin/bash -c " - cd /opt/hudi-rs/cpp && \ - cargo build --release && \ - cd /opt/hudi-rs/demo/file-group-api/cpp && \ - mkdir build && cd build && \ - cmake .. && \ - make && \ - ./file_group_api_cpp - " - -# Run the Rust and Python demo apps -# Note: no need to activate venv since this is already in a container -docker compose exec -T runner /bin/bash -c " - cd /opt/hudi-rs && \ - make setup develop && \ - cd /opt/hudi-rs/demo/sql-datafusion && ./run.sh &&\ - cd /opt/hudi-rs/demo/table-api-python && ./run.sh && \ - cd /opt/hudi-rs/demo/table-api-rust && ./run.sh - " diff --git a/demo/sql-datafusion/run.sh b/demo/sql-datafusion/run.sh deleted file mode 100755 index 2a6071f..0000000 --- a/demo/sql-datafusion/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cargo run -- --no-build --no-tests diff --git a/demo/table-api-python/run.sh b/demo/table-api-python/run.sh deleted file mode 100755 index e80a057..0000000 --- a/demo/table-api-python/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -python -m src.main diff --git a/demo/table-api-rust/run.sh b/demo/table-api-rust/run.sh deleted file mode 100755 index 2a6071f..0000000 --- a/demo/table-api-rust/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -cargo run -- --no-build --no-tests
