martin-g commented on code in PR #1375:
URL:
https://github.com/apache/datafusion-python/pull/1375#discussion_r2784595461
##########
.github/workflows/build.yml:
##########
@@ -15,61 +15,235 @@
# specific language governing permissions and limitations
# under the License.
-name: Python Release Build
+# Reusable workflow for running building
+# This ensures the same tests run for both debug (PRs) and release (main/tags)
builds
+
+name: Build
+
on:
- pull_request:
- branches: ["main"]
- push:
- tags: ["*-rc*"]
- branches: ["branch-*"]
+ workflow_call:
+ inputs:
+ build_mode:
+ description: 'Build mode: debug or release'
+ required: true
+ type: string
+ run_wheels:
+ description: 'Whether to build distribution wheels'
+ required: false
+ type: boolean
+ default: false
+
+env:
+ CARGO_TERM_COLOR: always
+ RUST_BACKTRACE: 1
jobs:
- build:
+ # ============================================
+ # Linting Jobs
+ # ============================================
+ lint-rust:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
+
+ - name: Setup Rust
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: "nightly"
+ components: rustfmt
+
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+
+ - name: Check formatting
+ run: cargo +nightly fmt --all -- --check
+
+ lint-python:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
- name: Install Python
- uses: actions/setup-python@v6
+ uses: actions/setup-python@v5
with:
python-version: "3.12"
- - uses: astral-sh/setup-uv@v7
+ - uses: astral-sh/setup-uv@v6
with:
- enable-cache: true
+ enable-cache: true
- # Use the --no-install-package to only install the dependencies
- # but do not yet build the rust library
- name: Install dependencies
run: uv sync --dev --no-install-package datafusion
- # Update output format to enable automatic inline annotations.
- name: Run Ruff
run: |
uv run --no-project ruff check --output-format=github python/
uv run --no-project ruff format --check python/
- name: Run codespell
run: |
- uv run --no-project codespell --toml pyproject.toml
+ uv run --no-project codespell --toml pyproject.toml
+
+ lint-toml:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Install taplo
+ uses: taiki-e/install-action@v2
+ with:
+ tool: taplo-cli
+
+ # if you encounter an error, try running 'taplo format' to fix the
formatting automatically.
+ - name: Check Cargo.toml formatting
+ run: taplo format --check
generate-license:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- - uses: astral-sh/setup-uv@v7
+
+ - uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+
+ - name: Install cargo-license
+ uses: taiki-e/install-action@v2
with:
- enable-cache: true
+ tool: cargo-license
- name: Generate license file
run: uv run --no-project python ./dev/create_license.py
+
- uses: actions/upload-artifact@v6
with:
name: python-wheel-license
path: LICENSE.txt
+ # ============================================
+ # Build - Linux x86_64
+ # ============================================
+ build-manylinux-x86_64:
+ needs: [generate-license, lint-rust, lint-python]
+ name: ManyLinux x86_64
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v7
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Setup Rust
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+ with:
+ key: ${{ inputs.build_mode }}
+
+ - uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+
+ - name: Build (release mode)
+ uses: PyO3/maturin-action@v1
+ if: inputs.build_mode == 'release'
+ with:
+ target: x86_64-unknown-linux-gnu
+ manylinux: "2_28"
+ args: --release --strip --features protoc,substrait --out dist
+ rustup-components: rust-std
+
+ - name: Build (debug mode)
+ uses: PyO3/maturin-action@v1
+ if: inputs.build_mode == 'debug'
+ with:
+ target: x86_64-unknown-linux-gnu
+ manylinux: "2_28"
+ args: --features protoc,substrait --out dist
+ rustup-components: rust-std
+
+ - name: Build FFI test library
+ uses: PyO3/maturin-action@v1
+ with:
+ target: x86_64-unknown-linux-gnu
+ manylinux: "2_28"
+ working-directory: examples/datafusion-ffi-example
+ args: --out dist
+ rustup-components: rust-std
+
+ - run: cp examples/datafusion-ffi-example/dist/*.whl dist/
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v6
+ with:
+ name: dist-manylinux-x86_64
+ path: dist/*
+
+ # ============================================
+ # Build - Linux ARM64
+ # ============================================
+ build-manylinux-aarch64:
+ needs: [generate-license, lint-rust, lint-python]
+ name: ManyLinux arm64
+ runs-on: ubuntu-latest
Review Comment:
It seems it reduced the time twice!
previous build (16 mins):
https://github.com/apache/datafusion-python/actions/runs/21838787774/job/63017243710
new one (8mins and half):
https://github.com/apache/datafusion-python/actions/runs/21840775031/job/63024275686?pr=1375
👍🏻
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
