This is an automated email from the ASF dual-hosted git repository.
timsaucer pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/main by this push:
new 3f89704f Build in debug mode for PRs (#1375)
3f89704f is described below
commit 3f89704f9d8885521a2d9f2f4c8e78d4a67e9b2a
Author: Tim Saucer <[email protected]>
AuthorDate: Wed Feb 11 07:34:53 2026 -0500
Build in debug mode for PRs (#1375)
* First draft of running debug mode for PRs and release mode for main &
releases
* Update paths
* Change install command for taplo
* install protoc
* taplo fmt
* Working through CI build issues
* More CI issues
* do not build taplo, just download it
* Try only running clippy when we can reuse build artifacts
* Try removing unnecessary installs during build
* Don't build cargo-license
* Add back in uv sync so we can run maturin
* minor: name casing
* Fix path for wheels
* More CI updates, but expect pytest to fail until we switch to downloading
the wheel artifacts from build stage
* Correct error in yml file. Rename to match other file extensions
* Download wheel from build stage for testing
* For CI tests move into test directory to avoid picking up pyproject.toml
file
* Do not upload artifacts not used in testing during debug builds
* Do not attempt to use local python path for tests
* Bump manylinux version
* Do not run release flow for branches named branch-*. Only run it for
pushes to main and release or candidate tags.
* Build FFI test code in build stage so we only build it once
* We need both wheels to be in the dist folder and the maturin action is
erasing the other wheel
* We now have two wheels that need to be installed instead of just one
* Make a minor change to restart CI
* tests will need both wheels also
* Update .github/workflows/build.yml
Co-authored-by: Martin Grigorov <[email protected]>
* mac has protoc system installed
* Update .github/workflows/test.yml
Co-authored-by: Copilot <[email protected]>
---------
Co-authored-by: Martin Grigorov <[email protected]>
Co-authored-by: Copilot <[email protected]>
---
.cargo/config.toml | 11 +-
.github/workflows/build.yml | 392 ++++++++++++++-------
.../pyproject.toml => .github/workflows/ci.yml | 36 +-
.../Cargo.toml => .github/workflows/release.yml | 49 +--
.github/workflows/{test.yaml => test.yml} | 71 ++--
Cargo.toml | 38 +-
dev/create_license.py | 4 +-
examples/datafusion-ffi-example/.cargo/config.toml | 11 +-
examples/datafusion-ffi-example/Cargo.toml | 16 +-
examples/datafusion-ffi-example/pyproject.toml | 6 +-
pyproject.toml | 195 +++++-----
11 files changed, 501 insertions(+), 328 deletions(-)
diff --git a/.cargo/config.toml b/.cargo/config.toml
index 91a099a6..af951327 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -1,12 +1,5 @@
[target.x86_64-apple-darwin]
-rustflags = [
- "-C", "link-arg=-undefined",
- "-C", "link-arg=dynamic_lookup",
-]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
[target.aarch64-apple-darwin]
-rustflags = [
- "-C", "link-arg=-undefined",
- "-C", "link-arg=dynamic_lookup",
-]
-
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 911c536a..8aeb86f1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -15,34 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-name: Python Release Build
+# Reusable workflow for running building
+# This ensures the same tests run for both debug (PRs) and release (main/tags)
builds
+
+name: Build
+
on:
- pull_request:
- branches: ["main"]
- push:
- tags: ["*-rc*"]
- branches: ["branch-*"]
+ workflow_call:
+ inputs:
+ build_mode:
+ description: 'Build mode: debug or release'
+ required: true
+ type: string
+ run_wheels:
+ description: 'Whether to build distribution wheels'
+ required: false
+ type: boolean
+ default: false
+
+env:
+ CARGO_TERM_COLOR: always
+ RUST_BACKTRACE: 1
jobs:
- build:
+ # ============================================
+ # Linting Jobs
+ # ============================================
+ lint-rust:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
+
+ - name: Setup Rust
+ uses: dtolnay/rust-toolchain@stable
+ with:
+ toolchain: "nightly"
+ components: rustfmt
+
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+
+ - name: Check formatting
+ run: cargo +nightly fmt --all -- --check
+
+ lint-python:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
- name: Install Python
- uses: actions/setup-python@v6
+ uses: actions/setup-python@v5
with:
python-version: "3.12"
- - uses: astral-sh/setup-uv@v7
+ - uses: astral-sh/setup-uv@v6
with:
- enable-cache: true
+ enable-cache: true
- # Use the --no-install-package to only install the dependencies
- # but do not yet build the rust library
- name: Install dependencies
run: uv sync --dev --no-install-package datafusion
- # Update output format to enable automatic inline annotations.
- name: Run Ruff
run: |
uv run --no-project ruff check --output-format=github python/
@@ -50,26 +82,168 @@ jobs:
- name: Run codespell
run: |
- uv run --no-project codespell --toml pyproject.toml
+ uv run --no-project codespell --toml pyproject.toml
+
+ lint-toml:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Install taplo
+ uses: taiki-e/install-action@v2
+ with:
+ tool: taplo-cli
+
+ # if you encounter an error, try running 'taplo format' to fix the
formatting automatically.
+ - name: Check Cargo.toml formatting
+ run: taplo format --check
generate-license:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- - uses: astral-sh/setup-uv@v7
+
+ - uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+
+ - name: Install cargo-license
+ uses: taiki-e/install-action@v2
with:
- enable-cache: true
+ tool: cargo-license
- name: Generate license file
run: uv run --no-project python ./dev/create_license.py
+
- uses: actions/upload-artifact@v6
with:
name: python-wheel-license
path: LICENSE.txt
+ # ============================================
+ # Build - Linux x86_64
+ # ============================================
+ build-manylinux-x86_64:
+ needs: [generate-license, lint-rust, lint-python]
+ name: ManyLinux x86_64
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v7
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Setup Rust
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+ with:
+ key: ${{ inputs.build_mode }}
+
+ - uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+
+ - name: Build (release mode)
+ uses: PyO3/maturin-action@v1
+ if: inputs.build_mode == 'release'
+ with:
+ target: x86_64-unknown-linux-gnu
+ manylinux: "2_28"
+ args: --release --strip --features protoc,substrait --out dist
+ rustup-components: rust-std
+
+ - name: Build (debug mode)
+ uses: PyO3/maturin-action@v1
+ if: inputs.build_mode == 'debug'
+ with:
+ target: x86_64-unknown-linux-gnu
+ manylinux: "2_28"
+ args: --features protoc,substrait --out dist
+ rustup-components: rust-std
+
+ - name: Build FFI test library
+ uses: PyO3/maturin-action@v1
+ with:
+ target: x86_64-unknown-linux-gnu
+ manylinux: "2_28"
+ working-directory: examples/datafusion-ffi-example
+ args: --out dist
+ rustup-components: rust-std
+
+ - run: cp examples/datafusion-ffi-example/dist/*.whl dist/
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v6
+ with:
+ name: dist-manylinux-x86_64
+ path: dist/*
+
+ # ============================================
+ # Build - Linux ARM64
+ # ============================================
+ build-manylinux-aarch64:
+ needs: [generate-license, lint-rust, lint-python]
+ name: ManyLinux arm64
+ runs-on: ubuntu-24.04-arm
+ steps:
+ - uses: actions/checkout@v6
+
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v7
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Setup Rust
+ uses: dtolnay/rust-toolchain@stable
+
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+ with:
+ key: ${{ inputs.build_mode }}
+
+ - uses: astral-sh/setup-uv@v6
+ with:
+ enable-cache: true
+
+ - name: Build (release mode)
+ uses: PyO3/maturin-action@v1
+ if: inputs.build_mode == 'release'
+ with:
+ target: aarch64-unknown-linux-gnu
+ manylinux: "2_28"
+ args: --release --strip --features protoc,substrait --out dist
+ rustup-components: rust-std
+
+ - name: Build (debug mode)
+ uses: PyO3/maturin-action@v1
+ if: inputs.build_mode == 'debug'
+ with:
+ target: aarch64-unknown-linux-gnu
+ manylinux: "2_28"
+ args: --features protoc,substrait --out dist
+ rustup-components: rust-std
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v6
+ if: inputs.build_mode == 'release'
+ with:
+ name: dist-manylinux-aarch64
+ path: dist/*
+
+ # ============================================
+ # Build - macOS arm64 / Windows
+ # ============================================
build-python-mac-win:
- needs: [generate-license]
- name: Mac/Win
+ needs: [generate-license, lint-rust, lint-python]
+ name: macOS arm64 & Windows
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
@@ -79,10 +253,6 @@ jobs:
steps:
- uses: actions/checkout@v6
- - uses: actions/setup-python@v6
- with:
- python-version: ${{ matrix.python-version }}
-
- uses: dtolnay/rust-toolchain@stable
- run: rm LICENSE.txt
@@ -92,20 +262,38 @@ jobs:
name: python-wheel-license
path: .
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+ with:
+ key: ${{ inputs.build_mode }}
+
+ - uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+
- name: Install Protoc
uses: arduino/setup-protoc@v3
with:
version: "27.4"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- - uses: astral-sh/setup-uv@v7
- with:
- enable-cache: true
+ - name: Install dependencies
+ run: uv sync --dev --no-install-package datafusion
- - name: Build Python package
- run: |
- uv sync --dev --no-install-package datafusion
- uv run --no-project maturin build --release --strip --features
substrait
+ # Run clippy BEFORE maturin so we can avoid rebuilding. The features
must match
+ # exactly the features used by maturin. Linux maturin builds need to
happen in a
+ # container so only run this for our mac runner.
+ - name: Run Clippy
+ if: matrix.os != 'windows-latest'
+ run: cargo clippy --no-deps --all-targets --features substrait -- -D
warnings
+
+ - name: Build Python package (release mode)
+ if: inputs.build_mode == 'release'
+ run: uv run --no-project maturin build --release --strip --features
substrait
+
+ - name: Build Python package (debug mode)
+ if: inputs.build_mode != 'release'
+ run: uv run --no-project maturin build --features substrait
- name: List Windows wheels
if: matrix.os == 'windows-latest'
@@ -120,13 +308,17 @@ jobs:
- name: Archive wheels
uses: actions/upload-artifact@v6
+ if: inputs.build_mode == 'release'
with:
name: dist-${{ matrix.os }}
path: target/wheels/*
+ # ============================================
+ # Build - macOS x86_64 (release only)
+ # ============================================
build-macos-x86_64:
- needs: [generate-license]
- name: Mac x86_64
+ if: inputs.build_mode == 'release'
+ needs: [generate-license, lint-rust, lint-python]
runs-on: macos-15-intel
strategy:
fail-fast: false
@@ -135,10 +327,6 @@ jobs:
steps:
- uses: actions/checkout@v6
- - uses: actions/setup-python@v6
- with:
- python-version: ${{ matrix.python-version }}
-
- uses: dtolnay/rust-toolchain@stable
- run: rm LICENSE.txt
@@ -148,19 +336,26 @@ jobs:
name: python-wheel-license
path: .
+ - name: Cache Cargo
+ uses: Swatinem/rust-cache@v2
+ with:
+ key: ${{ inputs.build_mode }}
+
+ - uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+
- name: Install Protoc
uses: arduino/setup-protoc@v3
with:
version: "27.4"
repo-token: ${{ secrets.GITHUB_TOKEN }}
- - uses: astral-sh/setup-uv@v7
- with:
- enable-cache: true
+ - name: Install dependencies
+ run: uv sync --dev --no-install-package datafusion
- - name: Build Python package
+ - name: Build (release mode)
run: |
- uv sync --dev --no-install-package datafusion
uv run --no-project maturin build --release --strip --features
substrait
- name: List Mac wheels
@@ -172,68 +367,14 @@ jobs:
name: dist-macos-aarch64
path: target/wheels/*
- build-manylinux-x86_64:
- needs: [generate-license]
- name: Manylinux x86_64
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v6
- - run: rm LICENSE.txt
- - name: Download LICENSE.txt
- uses: actions/download-artifact@v7
- with:
- name: python-wheel-license
- path: .
- - run: cat LICENSE.txt
- - name: Build wheels
- uses: PyO3/maturin-action@v1
- env:
- RUST_BACKTRACE: 1
- with:
- rust-toolchain: nightly
- target: x86_64
- manylinux: auto
- rustup-components: rust-std rustfmt # Keep them in one line due to
https://github.com/PyO3/maturin-action/issues/153
- args: --release --manylinux 2014 --features protoc,substrait
- - name: Archive wheels
- uses: actions/upload-artifact@v6
- with:
- name: dist-manylinux-x86_64
- path: target/wheels/*
-
- build-manylinux-aarch64:
- needs: [generate-license]
- name: Manylinux arm64
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v6
- - run: rm LICENSE.txt
- - name: Download LICENSE.txt
- uses: actions/download-artifact@v7
- with:
- name: python-wheel-license
- path: .
- - run: cat LICENSE.txt
- - name: Build wheels
- uses: PyO3/maturin-action@v1
- env:
- RUST_BACKTRACE: 1
- with:
- rust-toolchain: nightly
- target: aarch64
- # Use manylinux_2_28-cross because the manylinux2014-cross has GCC
4.8.5, which causes the build to fail
- manylinux: 2_28
- rustup-components: rust-std rustfmt # Keep them in one line due to
https://github.com/PyO3/maturin-action/issues/153
- args: --release --features protoc,substrait
- - name: Archive wheels
- uses: actions/upload-artifact@v6
- with:
- name: dist-manylinux-aarch64
- path: target/wheels/*
+ # ============================================
+ # Build - Source Distribution
+ # ============================================
build-sdist:
needs: [generate-license]
name: Source distribution
+ if: inputs.build_mode == 'release'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -253,16 +394,22 @@ jobs:
args: --release --sdist --out dist --features protoc,substrait
- name: Assert sdist build does not generate wheels
run: |
- if [ "$(ls -A target/wheels)" ]; then
- echo "Error: Sdist build generated wheels"
- exit 1
- else
- echo "Directory is clean"
- fi
+ if [ "$(ls -A target/wheels)" ]; then
+ echo "Error: Sdist build generated wheels"
+ exit 1
+ else
+ echo "Directory is clean"
+ fi
shell: bash
-
+
+ # ============================================
+ # Build - Source Distribution
+ # ============================================
+
merge-build-artifacts:
runs-on: ubuntu-latest
+ name: Merge build artifacts
+ if: inputs.build_mode == 'release'
needs:
- build-python-mac-win
- build-macos-x86_64
@@ -276,6 +423,9 @@ jobs:
name: dist
pattern: dist-*
+ # ============================================
+ # Build - Documentation
+ # ============================================
# Documentation build job that runs after wheels are built
build-docs:
name: Build docs
@@ -312,7 +462,7 @@ jobs:
- name: Setup Python
uses: actions/setup-python@v6
with:
- python-version: "3.11"
+ python-version: "3.10"
- name: Install dependencies
uses: astral-sh/setup-uv@v7
@@ -326,20 +476,21 @@ jobs:
name: dist-manylinux-x86_64
path: wheels/
- # Install from the pre-built wheel
- - name: Install from pre-built wheel
+ # Install from the pre-built wheels
+ - name: Install from pre-built wheels
run: |
set -x
uv venv
# Install documentation dependencies
uv sync --dev --no-install-package datafusion --group docs
- # Install the pre-built wheel
- WHEEL=$(find wheels/ -name "*.whl" | head -1)
- if [ -n "$WHEEL" ]; then
- echo "Installing wheel: $WHEEL"
- uv pip install "$WHEEL"
+ # Install all pre-built wheels
+ WHEELS=$(find wheels/ -name "*.whl")
+ if [ -n "$WHEELS" ]; then
+ echo "Installing wheels:"
+ echo "$WHEELS"
+ uv pip install wheels/*.whl
else
- echo "ERROR: No wheel found!"
+ echo "ERROR: No wheels found!"
exit 1
fi
@@ -368,16 +519,3 @@ jobs:
git commit -m 'Publish built docs triggered by ${{ github.sha }}'
git push || git push --force
fi
-
- # NOTE: PyPI publish needs to be done manually for now after release passed
the vote
- # release:
- # name: Publish in PyPI
- # needs: [build-manylinux, build-python-mac-win]
- # runs-on: ubuntu-latest
- # steps:
- # - uses: actions/download-artifact@v7
- # - name: Publish to PyPI
- # uses: pypa/gh-action-pypi-publish@master
- # with:
- # user: __token__
- # password: ${{ secrets.pypi_password }}
diff --git a/examples/datafusion-ffi-example/pyproject.toml
b/.github/workflows/ci.yml
similarity index 61%
copy from examples/datafusion-ffi-example/pyproject.toml
copy to .github/workflows/ci.yml
index 0c54df95..ab284b52 100644
--- a/examples/datafusion-ffi-example/pyproject.toml
+++ b/.github/workflows/ci.yml
@@ -15,19 +15,27 @@
# specific language governing permissions and limitations
# under the License.
-[build-system]
-requires = ["maturin>=1.6,<2.0"]
-build-backend = "maturin"
+# CI workflow for pull requests - runs tests in DEBUG mode for faster feedback
-[project]
-name = "datafusion_ffi_example"
-requires-python = ">=3.9"
-classifiers = [
- "Programming Language :: Rust",
- "Programming Language :: Python :: Implementation :: CPython",
- "Programming Language :: Python :: Implementation :: PyPy",
-]
-dynamic = ["version"]
+name: CI
-[tool.maturin]
-features = ["pyo3/extension-module"]
+on:
+ pull_request:
+ branches: ["main"]
+
+concurrency:
+ group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ uses: ./.github/workflows/build.yml
+ with:
+ build_mode: debug
+ run_wheels: false
+ secrets: inherit
+
+ test:
+ needs: build
+ uses: ./.github/workflows/test.yml
+ secrets: inherit
diff --git a/examples/datafusion-ffi-example/Cargo.toml
b/.github/workflows/release.yml
similarity index 52%
copy from examples/datafusion-ffi-example/Cargo.toml
copy to .github/workflows/release.yml
index e6708fce..bddc89ea 100644
--- a/examples/datafusion-ffi-example/Cargo.toml
+++ b/.github/workflows/release.yml
@@ -15,28 +15,35 @@
# specific language governing permissions and limitations
# under the License.
-[package]
-name = "datafusion-ffi-example"
-version = "0.2.0"
-edition = "2021"
+# Release workflow - runs tests in RELEASE mode and builds distribution wheels
+# Triggered on:
+# - Merges to main
+# - Release candidate tags (*-rc*)
+# - Release tags (e.g., 45.0.0)
-[dependencies]
-datafusion-catalog = { version = "52" , default-features = false }
-datafusion-common = { version = "52" , default-features = false }
-datafusion-functions-aggregate = { version = "52" }
-datafusion-functions-window = { version = "52" }
-datafusion-expr = { version = "52" }
-datafusion-ffi = { version = "52" }
+name: Release Build
-pyo3 = { version = "0.26", features = ["extension-module", "abi3",
"abi3-py39"] }
-arrow = { version = "57" }
-arrow-array = { version = "57" }
-arrow-schema = { version = "57" }
-async-trait = "0.1.89"
+on:
+ push:
+ branches:
+ - "main"
+ tags:
+ - "*-rc*" # Release candidates (e.g., 45.0.0-rc1)
+ - "[0-9]+.*" # Release tags (e.g., 45.0.0)
-[build-dependencies]
-pyo3-build-config = "0.26"
+concurrency:
+ group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
+ cancel-in-progress: true
-[lib]
-name = "datafusion_ffi_example"
-crate-type = ["cdylib", "rlib"]
+jobs:
+ build:
+ uses: ./.github/workflows/build.yml
+ with:
+ build_mode: release
+ run_wheels: true
+ secrets: inherit
+
+ test:
+ needs: build
+ uses: ./.github/workflows/test.yml
+ secrets: inherit
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yml
similarity index 76%
rename from .github/workflows/test.yaml
rename to .github/workflows/test.yml
index df4d8fcd..6fd6b02a 100644
--- a/.github/workflows/test.yaml
+++ b/.github/workflows/test.yml
@@ -15,16 +15,13 @@
# specific language governing permissions and limitations
# under the License.
-name: Python test
-on:
- push:
- branches: [main]
- pull_request:
- branches: [main]
+# Reusable workflow for running tests
+# This ensures the same tests run for both debug (PRs) and release (main/tags)
builds
+
+name: Test
-concurrency:
- group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
- cancel-in-progress: true
+on:
+ workflow_call:
jobs:
test-matrix:
@@ -50,7 +47,7 @@ jobs:
EXAMPLE_VERSION=$(grep -A 1 "name = \"datafusion-common\""
examples/datafusion-ffi-example/Cargo.lock | grep "version = " | head -1 | sed
's/.*version = "\(.*\)"/\1/')
echo "Main crate datafusion version: $MAIN_VERSION"
echo "FFI example datafusion version: $EXAMPLE_VERSION"
-
+
if [ "$MAIN_VERSION" != "$EXAMPLE_VERSION" ]; then
echo "❌ Error: FFI example datafusion versions don't match!"
exit 1
@@ -59,8 +56,6 @@ jobs:
- name: Setup Rust Toolchain
uses: dtolnay/rust-toolchain@stable
id: rust-toolchain
- with:
- components: clippy,rustfmt
- name: Install Protoc
uses: arduino/setup-protoc@v3
@@ -79,28 +74,46 @@ jobs:
path: ~/.cargo
key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{
hashFiles('Cargo.lock') }}
- - name: Run Clippy
- if: ${{ matrix.python-version == '3.10' && matrix.toolchain ==
'stable' }}
- run: cargo clippy --all-targets --all-features -- -D clippy::all -D
warnings -A clippy::redundant_closure
-
- - name: Install dependencies and build
+ - name: Install dependencies
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
+ # Download the Linux wheel built in the build workflow
+ - name: Download pre-built Linux wheel
+ uses: actions/download-artifact@v7
+ with:
+ name: dist-manylinux-x86_64
+ path: wheels/
+
+ # Install from the pre-built wheels
+ - name: Install from pre-built wheels
+ run: |
+ set -x
+ uv venv
+ # Install development dependencies
+ uv sync --dev --no-install-package datafusion
+ # Install all pre-built wheels
+ WHEELS=$(find wheels/ -name "*.whl")
+ if [ -n "$WHEELS" ]; then
+ echo "Installing wheels:"
+ echo "$WHEELS"
+ uv pip install wheels/*.whl
+ else
+ echo "ERROR: No wheels found!"
+ exit 1
+ fi
+
- name: Run tests
env:
RUST_BACKTRACE: 1
run: |
git submodule update --init
- uv sync --dev --no-install-package datafusion
- uv run --no-project maturin develop --uv
- uv run --no-project pytest -v .
+ uv run --no-project pytest -v . --import-mode=importlib
- name: FFI unit tests
run: |
cd examples/datafusion-ffi-example
- uv run --no-project maturin develop --uv
uv run --no-project pytest python/tests/_test*.py
- name: Cache the generated dataset
@@ -121,19 +134,3 @@ jobs:
cd examples/tpch
uv run --no-project python convert_data_to_parquet.py
uv run --no-project pytest _tests.py
-
- nightly-fmt:
- runs-on: ubuntu-latest
-
- steps:
- - uses: actions/checkout@v6
-
- - name: Setup Rust Toolchain
- uses: dtolnay/rust-toolchain@stable
- id: rust-toolchain
- with:
- toolchain: "nightly"
- components: clippy,rustfmt
-
- - name: Check Formatting
- run: cargo +nightly fmt -- --check
diff --git a/Cargo.toml b/Cargo.toml
index af2ffb01..f4e8575c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,13 +27,13 @@ license = "Apache-2.0"
edition = "2021"
rust-version = "1.78"
include = [
- "/src",
- "/datafusion",
- "/LICENSE.txt",
- "build.rs",
- "pyproject.toml",
- "Cargo.toml",
- "Cargo.lock",
+ "/src",
+ "/datafusion",
+ "/LICENSE.txt",
+ "build.rs",
+ "pyproject.toml",
+ "Cargo.toml",
+ "Cargo.lock",
]
[features]
@@ -43,15 +43,15 @@ substrait = ["dep:datafusion-substrait"]
[dependencies]
tokio = { version = "1.47", features = [
- "macros",
- "rt",
- "rt-multi-thread",
- "sync",
+ "macros",
+ "rt",
+ "rt-multi-thread",
+ "sync",
] }
pyo3 = { version = "0.26", features = [
- "extension-module",
- "abi3",
- "abi3-py310",
+ "extension-module",
+ "abi3",
+ "abi3-py310",
] }
pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
pyo3-log = "0.13.2"
@@ -64,16 +64,16 @@ datafusion-ffi = { version = "52" }
prost = "0.14.1" # keep in line with `datafusion-substrait`
uuid = { version = "1.18", features = ["v4"] }
mimalloc = { version = "0.1", optional = true, default-features = false,
features = [
- "local_dynamic_tls",
+ "local_dynamic_tls",
] }
async-trait = "0.1.89"
futures = "0.3"
cstr = "0.2"
object_store = { version = "0.12.4", features = [
- "aws",
- "gcp",
- "azure",
- "http",
+ "aws",
+ "gcp",
+ "azure",
+ "http",
] }
url = "2"
log = "0.4.27"
diff --git a/dev/create_license.py b/dev/create_license.py
index a28a0abe..acbf8587 100644
--- a/dev/create_license.py
+++ b/dev/create_license.py
@@ -22,11 +22,9 @@ import json
import subprocess
from pathlib import Path
-subprocess.check_output(["cargo", "install", "cargo-license"])
data = subprocess.check_output(
[
- "cargo",
- "license",
+ "cargo-license",
"--avoid-build-deps",
"--avoid-dev-deps",
"--do-not-bundle",
diff --git a/examples/datafusion-ffi-example/.cargo/config.toml
b/examples/datafusion-ffi-example/.cargo/config.toml
index 91a099a6..af951327 100644
--- a/examples/datafusion-ffi-example/.cargo/config.toml
+++ b/examples/datafusion-ffi-example/.cargo/config.toml
@@ -1,12 +1,5 @@
[target.x86_64-apple-darwin]
-rustflags = [
- "-C", "link-arg=-undefined",
- "-C", "link-arg=dynamic_lookup",
-]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
[target.aarch64-apple-darwin]
-rustflags = [
- "-C", "link-arg=-undefined",
- "-C", "link-arg=dynamic_lookup",
-]
-
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
diff --git a/examples/datafusion-ffi-example/Cargo.toml
b/examples/datafusion-ffi-example/Cargo.toml
index e6708fce..d54add58 100644
--- a/examples/datafusion-ffi-example/Cargo.toml
+++ b/examples/datafusion-ffi-example/Cargo.toml
@@ -21,14 +21,18 @@ version = "0.2.0"
edition = "2021"
[dependencies]
-datafusion-catalog = { version = "52" , default-features = false }
-datafusion-common = { version = "52" , default-features = false }
-datafusion-functions-aggregate = { version = "52" }
-datafusion-functions-window = { version = "52" }
-datafusion-expr = { version = "52" }
+datafusion-catalog = { version = "52", default-features = false }
+datafusion-common = { version = "52", default-features = false }
+datafusion-functions-aggregate = { version = "52" }
+datafusion-functions-window = { version = "52" }
+datafusion-expr = { version = "52" }
datafusion-ffi = { version = "52" }
-pyo3 = { version = "0.26", features = ["extension-module", "abi3",
"abi3-py39"] }
+pyo3 = { version = "0.26", features = [
+ "extension-module",
+ "abi3",
+ "abi3-py39",
+] }
arrow = { version = "57" }
arrow-array = { version = "57" }
arrow-schema = { version = "57" }
diff --git a/examples/datafusion-ffi-example/pyproject.toml
b/examples/datafusion-ffi-example/pyproject.toml
index 0c54df95..7f85e948 100644
--- a/examples/datafusion-ffi-example/pyproject.toml
+++ b/examples/datafusion-ffi-example/pyproject.toml
@@ -23,9 +23,9 @@ build-backend = "maturin"
name = "datafusion_ffi_example"
requires-python = ">=3.9"
classifiers = [
- "Programming Language :: Rust",
- "Programming Language :: Python :: Implementation :: CPython",
- "Programming Language :: Python :: Implementation :: PyPy",
+ "Programming Language :: Rust",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy",
]
dynamic = ["version"]
diff --git a/pyproject.toml b/pyproject.toml
index 497943a3..d315dbe1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -27,26 +27,26 @@ license = { file = "LICENSE.txt" }
requires-python = ">=3.10"
keywords = ["datafusion", "dataframe", "rust", "query-engine"]
classifiers = [
- "Development Status :: 2 - Pre-Alpha",
- "Intended Audience :: Developers",
- "License :: OSI Approved :: Apache Software License",
- "License :: OSI Approved",
- "Operating System :: MacOS",
- "Operating System :: Microsoft :: Windows",
- "Operating System :: POSIX :: Linux",
- "Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Programming Language :: Python :: 3.13",
- "Programming Language :: Python :: 3.14",
- "Programming Language :: Python",
- "Programming Language :: Rust",
+ "Development Status :: 2 - Pre-Alpha",
+ "Intended Audience :: Developers",
+ "License :: OSI Approved :: Apache Software License",
+ "License :: OSI Approved",
+ "Operating System :: MacOS",
+ "Operating System :: Microsoft :: Windows",
+ "Operating System :: POSIX :: Linux",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+ "Programming Language :: Python :: 3.14",
+ "Programming Language :: Python",
+ "Programming Language :: Rust",
]
dependencies = [
- "pyarrow>=16.0.0;python_version<'3.14'",
- "pyarrow>=22.0.0;python_version>='3.14'",
- "typing-extensions;python_version<'3.13'"
+ "pyarrow>=16.0.0;python_version<'3.14'",
+ "pyarrow>=22.0.0;python_version>='3.14'",
+ "typing-extensions;python_version<'3.13'",
]
dynamic = ["version"]
@@ -73,23 +73,23 @@ asyncio_default_fixture_loop_scope = "function"
# Enable docstring linting using the google style guide
[tool.ruff.lint]
-select = ["ALL" ]
+select = ["ALL"]
ignore = [
- "A001", # Allow using words like min as variable names
- "A002", # Allow using words like filter as variable names
- "ANN401", # Allow Any for wrapper classes
- "COM812", # Recommended to ignore these rules when using with ruff-format
- "FIX002", # Allow TODO lines - consider removing at some point
- "FBT001", # Allow boolean positional args
- "FBT002", # Allow boolean positional args
- "ISC001", # Recommended to ignore these rules when using with ruff-format
- "SLF001", # Allow accessing private members
- "TD002", # Do not require author names in TODO statements
- "TD003", # Allow TODO lines
- "PLR0913", # Allow many arguments in function definition
- "PD901", # Allow variable name df
- "N812", # Allow importing functions as `F`
- "A005", # Allow module named io
+ "A001", # Allow using words like min as variable names
+ "A002", # Allow using words like filter as variable names
+ "ANN401", # Allow Any for wrapper classes
+ "COM812", # Recommended to ignore these rules when using with ruff-format
+ "FIX002", # Allow TODO lines - consider removing at some point
+ "FBT001", # Allow boolean positional args
+ "FBT002", # Allow boolean positional args
+ "ISC001", # Recommended to ignore these rules when using with ruff-format
+ "SLF001", # Allow accessing private members
+ "TD002", # Do not require author names in TODO statements
+ "TD003", # Allow TODO lines
+ "PLR0913", # Allow many arguments in function definition
+ "PD901", # Allow variable name df
+ "N812", # Allow importing functions as `F`
+ "A005", # Allow module named io
]
[tool.ruff.lint.pydocstyle]
@@ -104,61 +104,96 @@ extend-allowed-calls = ["lit", "datafusion.lit"]
# Disable docstring checking for these directories
[tool.ruff.lint.per-file-ignores]
"python/tests/*" = [
- "ANN",
- "ARG",
- "BLE001",
- "D",
- "S101",
- "SLF",
- "PD",
- "PLR2004",
- "PT011",
- "RUF015",
- "S608",
- "PLR0913",
- "PT004",
+ "ANN",
+ "ARG",
+ "BLE001",
+ "D",
+ "S101",
+ "SLF",
+ "PD",
+ "PLR2004",
+ "PT011",
+ "RUF015",
+ "S608",
+ "PLR0913",
+ "PT004",
+]
+"examples/*" = [
+ "D",
+ "W505",
+ "E501",
+ "T201",
+ "S101",
+ "PLR2004",
+ "ANN001",
+ "ANN202",
+ "INP001",
+ "DTZ007",
+ "RUF015",
+]
+"dev/*" = [
+ "D",
+ "E",
+ "T",
+ "S",
+ "PLR",
+ "C",
+ "SIM",
+ "UP",
+ "EXE",
+ "N817",
+ "ERA001",
+ "ANN001",
+]
+"benchmarks/*" = [
+ "D",
+ "F",
+ "T",
+ "BLE",
+ "FURB",
+ "PLR",
+ "E",
+ "TD",
+ "TRY",
+ "S",
+ "SIM",
+ "EXE",
+ "UP",
+ "ERA001",
+ "ANN001",
+ "INP001",
]
-"examples/*" = ["D", "W505", "E501", "T201", "S101", "PLR2004", "ANN001",
"ANN202", "INP001", "DTZ007", "RUF015"]
-"dev/*" = ["D", "E", "T", "S", "PLR", "C", "SIM", "UP", "EXE", "N817",
"ERA001", "ANN001"]
-"benchmarks/*" = ["D", "F", "T", "BLE", "FURB", "PLR", "E", "TD", "TRY", "S",
"SIM", "EXE", "UP", "ERA001", "ANN001", "INP001"]
"docs/*" = ["D"]
"docs/source/conf.py" = ["ERA001", "ANN001", "INP001"]
[tool.codespell]
-skip = [
- "./target",
- "uv.lock",
- "./python/tests/test_functions.py"
-]
+skip = ["./target", "uv.lock", "./python/tests/test_functions.py"]
count = true
-ignore-words-list = [
- "ans",
- "IST"
-]
+ignore-words-list = ["ans", "IST"]
[dependency-groups]
dev = [
- "maturin>=1.8.1",
- "numpy>1.25.0;python_version<'3.14'",
- "numpy>=2.3.2;python_version>='3.14'",
- "pyarrow>=19.0.0",
- "pre-commit>=4.3.0",
- "pyyaml>=6.0.3",
- "pytest>=7.4.4",
- "pytest-asyncio>=0.23.3",
- "ruff>=0.9.1",
- "toml>=0.10.2",
- "pygithub==2.5.0",
- "codespell==2.4.1",
+ "maturin>=1.8.1",
+ "numpy>1.25.0;python_version<'3.14'",
+ "numpy>=2.3.2;python_version>='3.14'",
+ "pyarrow>=19.0.0",
+ "pre-commit>=4.3.0",
+ "pyyaml>=6.0.3",
+ "pytest>=7.4.4",
+ "pytest-asyncio>=0.23.3",
+ "ruff>=0.9.1",
+ "toml>=0.10.2",
+ "pygithub==2.5.0",
+ "codespell==2.4.1",
]
docs = [
- "sphinx>=7.1.2",
- "pydata-sphinx-theme==0.8.0",
- "myst-parser>=3.0.1",
- "jinja2>=3.1.5",
- "ipython>=8.12.3",
- "pandas>=2.0.3",
- "pickleshare>=0.7.5",
- "sphinx-autoapi>=3.4.0",
- "setuptools>=75.3.0",
+ "sphinx>=7.1.2",
+ "pydata-sphinx-theme==0.8.0",
+ "myst-parser>=3.0.1",
+ "jinja2>=3.1.5",
+ "ipython>=8.12.3",
+ "pandas>=2.0.3",
+ "pickleshare>=0.7.5",
+ "sphinx-autoapi>=3.4.0",
+ "setuptools>=75.3.0",
]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
