This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 3559a7ce56 [python] Centralize PyPaimon packaging configuration
(#10241)
3559a7ce56 is described below
commit 3559a7ce56b9d42305d513b381f8525b5ed08c88
Author: Zouxxyy <[email protected]>
AuthorDate: Sat Oct 3 17:04:38 2026 +0800
[python] Centralize PyPaimon packaging configuration (#10241)
---
.github/workflows/ci-python.yml | 153 ++++-----
.github/workflows/release-python-publish.yml | 2 +-
.github/workflows/release-python.yml | 16 +-
.github/workflows/release.yml | 2 +-
.gitignore | 1 +
docs/docs/project/creating-a-release.md | 9 +-
docs/docs/project/releases.md | 2 +-
docs/docs/project/verifying-a-release-candidate.md | 14 +-
docs/docs/pypaimon/installation.md | 18 +-
paimon-python/MANIFEST.in | 2 +-
paimon-python/README.md | 37 ++-
paimon-python/conftest.py | 3 +-
paimon-python/dev/check-licensing.sh | 2 +-
paimon-python/dev/install.py | 71 +++++
paimon-python/dev/lint-python.sh | 6 +-
paimon-python/dev/requirements-dev.txt | 41 ---
paimon-python/dev/requirements.txt | 44 ---
paimon-python/dev/run_mixed_tests.sh | 25 +-
paimon-python/dev/test_packaging.py | 212 +++++++++++++
.../{MANIFEST.in => pypaimon/_version.py} | 14 +-
paimon-python/pypaimon/build_info.py | 48 ++-
paimon-python/pypaimon/tests/build_info_test.py | 7 +-
.../tests/ray_data_evolution_merge_into_test.py | 3 +-
.../pypaimon/tests/rest/rest_read_write_test.py | 17 +-
paimon-python/pyproject.toml | 211 +++++++++++++
paimon-python/setup.py | 341 +++------------------
26 files changed, 704 insertions(+), 597 deletions(-)
diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml
index fddb1a4a6d..952a1173a5 100644
--- a/.github/workflows/ci-python.yml
+++ b/.github/workflows/ci-python.yml
@@ -27,7 +27,6 @@ permissions:
env:
JDK_VERSION: 8
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30
-Dmaven.wagon.http.retryHandler.requestSentEnabled=true
- LUMINA_DATA_VERSION: 0.1.0
jobs:
@@ -95,19 +94,26 @@ jobs:
- name: Verify Python version
run: python --version
- - name: Verify requirements.txt dependencies can be installed
+ - name: Verify Python distributions
+ run: |
+ # Build tools and legacy Flake8 require incompatible
importlib-metadata versions.
+ python -m venv "${RUNNER_TEMP}/paimon-packaging"
+ "${RUNNER_TEMP}/paimon-packaging/bin/python" -m pip install
--upgrade pip setuptools wheel packaging tomli build
+ "${RUNNER_TEMP}/paimon-packaging/bin/python"
paimon-python/dev/test_packaging.py
+
+ - name: Verify core package dependencies can be installed
shell: bash
run: |
cd paimon-python
python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| {
- echo "ERROR: Failed to resolve dependencies from
dev/requirements.txt"
+ python -m pip install . --target "$TEMP_DIR" || {
+ echo "ERROR: Failed to install core PyPaimon dependencies"
rm -rf "$TEMP_DIR"
exit 1
}
rm -rf "$TEMP_DIR"
- echo "✓ dev/requirements.txt can be resolved on this Python"
+ echo "✓ Core PyPaimon dependencies resolve on this Python"
- name: Build Java
run: |
@@ -118,57 +124,20 @@ jobs:
- name: Install Python dependencies
shell: bash
run: |
- df -h
- if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
- python -m pip install --upgrade pip==21.3.1
- python --version
- python -m pip install --no-cache-dir pyroaring
readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4'
'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12'
'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest
py4j==0.10.9.9 requests parameterized==0.8.1 datasketches==4.1.0 2>&1 >/dev/null
- python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION
}}' -i https://pypi.org/simple/
- elif [[ "${{ matrix.python-version }}" == "3.7" ]]; then
- # 3.7 installs the version-pinned set declared for 3.7 in
dev/requirements.txt.
- # ray/lance/daft/torch have no 3.7 wheels; those tests are
importorskip-ed (as on 3.6.15).
- python -m pip install --upgrade pip
- python -m pip install -r paimon-python/dev/requirements.txt
- python -m pip install flake8==4.0.1 'pytest~=7.0' py4j==0.10.9.9
parameterized==0.9.0 datasketches==4.1.0
- python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION
}}' -i https://pypi.org/simple/
+ python -m pip install --upgrade pip
+ if python -c "import sys; sys.exit(0 if sys.version_info < (3, 8)
else 1)"; then
+ # Old pip has no --group; read the same TOML with the
compatibility bridge.
+ python -m pip install tomli
+ python paimon-python/dev/install.py legacy
else
- python -m pip install --upgrade pip
- pip install torch --index-url https://download.pytorch.org/whl/cpu
- python -m pip install pyroaring readerwriterlock==1.0.9
fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0
fastavro==1.11.1 'isal>=1.8,<2' zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2
pylance==0.39.0 cramjam pytest~=7.0 py4j==0.10.9.9 requests
parameterized==0.9.0 'daft>=0.7.6' 'datafusion>=54,<55' datasketches
- if python -c "import sys; sys.exit(0 if sys.version_info >= (3,
12) else 1)"; then
- python -m pip install pyarrow==24.0.0 numpy==2.4.6 pandas==2.3.3
flake8==7.1.2
- else
- python -m pip install pyarrow==16.0.0 numpy==1.24.3
pandas==2.0.3 flake8==4.0.1
- fi
- python -m pip install 'h5py>=3,<4' Pillow
- python -c "import h5py; print('h5py', h5py.__version__)"
- if python -c "import sys; sys.exit(0 if sys.version_info >= (3,
10) else 1)"; then
- python -m pip install './paimon-python[rosbag]'
- python -c "import rosbags; print('rosbags installed')"
- fi
+ python -m pip install torch --index-url
https://download.pytorch.org/whl/cpu
+ dependency_group=dev
if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
- python -m pip install './paimon-python[lerobot,video]'
- python -c "import datasets, lerobot; print('datasets',
datasets.__version__, 'lerobot', lerobot.__version__)"
- fi
-
- # Required by SQL tests; native planner tests run against main
below.
- python -m pip install pypaimon-rust
- if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
- # Run the RoboMIND pipeline tests with synthetic local HDF5 data.
- python -m pip install "./paimon-python[sql]"
- fi
- python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION
}}' -i https://pypi.org/simple/
- if python -c "import sys; sys.exit(0 if sys.version_info >= (3,
11) else 1)"; then
- # Vortex requires pyarrow >= 17, while Pypaimon requires pyarrow
< 20 and != 19.
- python -m pip install vortex-data==0.70.0 pyarrow==18.1.0
- fi
- python -m pip install 'paimon-mosaic>=0.1.0'
- if [[ "${{ matrix.python-version }}" == "3.11" ]]; then
- python -m pip check
+ dependency_group=torch
fi
+ python -m pip install -e ./paimon-python --group
"./paimon-python/pyproject.toml:${dependency_group}"
fi
- python -m pip install 'oss2>=2.18,<3'
- df -h
+ python -m pip check
- name: Run lint-python.sh
shell: bash
@@ -204,13 +173,7 @@ jobs:
- name: Install Python test dependencies
run: |
python -m pip install torch --index-url
https://download.pytorch.org/whl/cpu
- python -m pip install "./paimon-python[sql,rosbag]" \
- readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3
ossfs==2023.12.0 \
- ray==2.54.0 fastavro==1.11.1 zstandard==0.24.0 polars==1.32.0
duckdb==1.3.2 \
- pylance==0.39.0 'pytest~=7.0' py4j==0.10.9.9 parameterized==0.9.0 \
- 'daft>=0.7.6' datasketches pyarrow==18.1.0 numpy==1.24.3
pandas==2.0.3 \
- 'h5py>=3,<4' Pillow 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' \
- vortex-data==0.70.0 'paimon-mosaic>=0.1.0'
+ python -m pip install -e ./paimon-python --group
./paimon-python/pyproject.toml:dev
python -m pip check
- name: Verify native wheel provenance
@@ -262,21 +225,20 @@ jobs:
run: python --version
- name: Install Python dependencies
- shell: bash
run: |
- python -m pip install --upgrade pip
- pip install torch --index-url https://download.pytorch.org/whl/cpu
- python -m pip install pyroaring readerwriterlock==1.0.9
fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.54.0
fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2
numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0
py4j==0.10.9.9 requests parameterized==0.9.0 datasketches 'datasets>=4,<4.1'
- python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION
}}' -i https://pypi.org/simple/
+ python -m pip install --upgrade pip
+ python -m pip install torch --index-url
https://download.pytorch.org/whl/cpu
+ python -m pip install -e ./paimon-python --group
./paimon-python/pyproject.toml:torch
+ python -m pip check
- name: Run lint-python.sh
shell: bash
run: |
chmod +x paimon-python/dev/lint-python.sh
./paimon-python/dev/lint-python.sh -i pytest_torch
- # One job: check dev/requirements.txt on each Python version in sequence,
then Ray version test on 3.10.
+ # Check core installation across Python versions, then basic Ray
compatibility on 3.10.
compatibility:
- name: Requirements and Ray compatibility
+ name: Package and Ray compatibility
timeout-minutes: 60
runs-on: ubuntu-latest
steps:
@@ -289,71 +251,71 @@ jobs:
build-essential git curl cmake \
&& sudo rm -rf /var/lib/apt/lists/*
- - name: Verify dev/requirements.txt on Python 3.8
+ - name: Verify core installation on Python 3.8
uses: actions/setup-python@v5
with:
python-version: '3.8'
- - name: Resolve requirements (3.8)
+ - name: Install core package (3.8)
run: |
cd paimon-python && python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| { rm -rf "$TEMP_DIR"; exit 1; }
- rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.8"
+ python -m pip install . --target "$TEMP_DIR" || { rm -rf
"$TEMP_DIR"; exit 1; }
+ rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.8"
- - name: Verify dev/requirements.txt on Python 3.9
+ - name: Verify core installation on Python 3.9
uses: actions/setup-python@v5
with:
python-version: '3.9'
- - name: Resolve requirements (3.9)
+ - name: Install core package (3.9)
run: |
cd paimon-python && python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| { rm -rf "$TEMP_DIR"; exit 1; }
- rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.9"
+ python -m pip install . --target "$TEMP_DIR" || { rm -rf
"$TEMP_DIR"; exit 1; }
+ rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.9"
- - name: Verify dev/requirements.txt on Python 3.10
+ - name: Verify core installation on Python 3.10
uses: actions/setup-python@v5
with:
python-version: '3.10'
- - name: Resolve requirements (3.10)
+ - name: Install core package (3.10)
run: |
cd paimon-python && python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| { rm -rf "$TEMP_DIR"; exit 1; }
- rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.10"
+ python -m pip install . --target "$TEMP_DIR" || { rm -rf
"$TEMP_DIR"; exit 1; }
+ rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.10"
- - name: Verify dev/requirements.txt on Python 3.11
+ - name: Verify core installation on Python 3.11
uses: actions/setup-python@v5
with:
python-version: '3.11'
- - name: Resolve requirements (3.11)
+ - name: Install core package (3.11)
run: |
cd paimon-python && python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| { rm -rf "$TEMP_DIR"; exit 1; }
- rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.11"
+ python -m pip install . --target "$TEMP_DIR" || { rm -rf
"$TEMP_DIR"; exit 1; }
+ rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.11"
- - name: Verify dev/requirements.txt on Python 3.12
+ - name: Verify core installation on Python 3.12
uses: actions/setup-python@v5
with:
python-version: '3.12'
- - name: Resolve requirements (3.12)
+ - name: Install core package (3.12)
run: |
cd paimon-python && python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| { rm -rf "$TEMP_DIR"; exit 1; }
- rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.12"
+ python -m pip install . --target "$TEMP_DIR" || { rm -rf
"$TEMP_DIR"; exit 1; }
+ rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.12"
- - name: Verify dev/requirements.txt on Python 3.13
+ - name: Verify core installation on Python 3.13
uses: actions/setup-python@v5
with:
python-version: '3.13'
- - name: Resolve requirements (3.13)
+ - name: Install core package (3.13)
run: |
cd paimon-python && python -m pip install --upgrade pip
TEMP_DIR=$(mktemp -d)
- python -m pip install -r dev/requirements.txt --target "$TEMP_DIR"
|| { rm -rf "$TEMP_DIR"; exit 1; }
- rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.13"
+ python -m pip install . --target "$TEMP_DIR" || { rm -rf
"$TEMP_DIR"; exit 1; }
+ rm -rf "$TEMP_DIR" && echo "✓ Core package OK on Python 3.13"
- name: Setup Python 3.10 for Ray test
uses: actions/setup-python@v5
@@ -362,13 +324,10 @@ jobs:
- name: Install base Python dependencies
run: |
python -m pip install --upgrade pip
- pip install torch --index-url https://download.pytorch.org/whl/cpu
- python -m pip install --no-cache-dir \
- pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1
cachetools==5.3.3 ossfs==2023.12.0 \
- fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0
duckdb==1.3.2 \
- numpy==1.24.3 pandas==2.0.3 cramjam pytest~=7.0 py4j==0.10.9.9
requests \
- parameterized==0.9.0 packaging datasketches
- python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}'
-i https://pypi.org/simple/
+ python -m pip install torch --index-url
https://download.pytorch.org/whl/cpu
+ # Use only test tools here: the dev group's Ray >=2.50 floor is for
full-feature tests.
+ python -m pip install
'./paimon-python[ray,oss,lumina,vindex,full-text,theta-sketch]' \
+ --group ./paimon-python/pyproject.toml:test
- name: Test Ray version compatibility
run: |
cd paimon-python
diff --git a/.github/workflows/release-python-publish.yml
b/.github/workflows/release-python-publish.yml
index a153ff2d4c..bb0fec4b70 100644
--- a/.github/workflows/release-python-publish.yml
+++ b/.github/workflows/release-python-publish.yml
@@ -61,7 +61,7 @@ jobs:
source_version="$(
sed -n 's/^VERSION = "\(.*\)"/\1/p' \
- paimon-python/setup.py
+ paimon-python/pypaimon/_version.py
)"
release_version="${{ inputs.release_version }}"
if [[ "${source_version}" != "${release_version}" ]]; then
diff --git a/.github/workflows/release-python.yml
b/.github/workflows/release-python.yml
index c8d3ef32e2..f0bab24cbf 100644
--- a/.github/workflows/release-python.yml
+++ b/.github/workflows/release-python.yml
@@ -52,7 +52,7 @@ jobs:
with:
python-version: "3.11"
cache: pip
- cache-dependency-path: paimon-python/dev/requirements.txt
+ cache-dependency-path: paimon-python/pyproject.toml
- name: Set package version
id: package_version
@@ -62,7 +62,7 @@ jobs:
source_version="$(
sed -n 's/^VERSION = "\(.*\)"/\1/p' \
- paimon-python/setup.py
+ paimon-python/pypaimon/_version.py
)"
release_version="${{ inputs.release_version }}"
if [[ "${source_version}" != "${release_version}" ]]; then
@@ -74,19 +74,18 @@ jobs:
if [[ "${GITHUB_REF}" == refs/tags/*-rc* ]]; then
rc_number="${GITHUB_REF_NAME##*-rc}"
package_version="${release_version}rc${rc_number}"
- sed -i \
- "s/^VERSION = \".*\"/VERSION = \"${package_version}\"/" \
- paimon-python/setup.py
fi
echo "version=${package_version}" >> "${GITHUB_OUTPUT}"
+ echo "PYPAIMON_BUILD_VERSION=${package_version}" >> "${GITHUB_ENV}"
echo "Building PyPaimon ${package_version}"
- name: Build distributions
working-directory: paimon-python
run: |
- python -m pip install --upgrade pip setuptools wheel twine
- python setup.py sdist bdist_wheel
+ python -m pip install --upgrade pip
+ python -m pip install --group build
+ python -m build
python -m twine check dist/*
- name: Verify distribution contents
@@ -111,7 +110,8 @@ jobs:
with tarfile.open(sdist, "r:gz") as archive:
names = archive.getnames()
prefix = "pypaimon-{}/".format(expected)
- for required in ("LICENSE", "NOTICE", "README.md", "setup.py"):
+ for required in ("LICENSE", "NOTICE", "README.md", "setup.py",
"pyproject.toml",
+ "pypaimon/_version.py",
"pypaimon/_full_version"):
if prefix + required not in names:
raise SystemExit("Missing {} from
sdist".format(required))
sample_data = "pypaimon/sample/data/data.jsonl"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index caaf86a89e..2316ae6968 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -81,7 +81,7 @@ jobs:
python_release_version="$(
sed -n 's/^VERSION = "\(.*\)"/\1/p' \
- paimon-python/setup.py
+ paimon-python/pypaimon/_version.py
)"
if [[ "${python_release_version}" != "${release_version}" ]]; then
echo "PyPaimon version ${python_release_version:-missing} does not
match Paimon ${release_version}" >&2
diff --git a/.gitignore b/.gitignore
index 54381f82eb..9f1331ac3d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ metastore_db/
### Python ###
*.pyc
+paimon-python/build/
paimon-python/dist/
paimon-python/*.egg-info/
paimon-python/dev/log
diff --git a/docs/docs/project/creating-a-release.md
b/docs/docs/project/creating-a-release.md
index c8c15c41e2..f42e2060c5 100644
--- a/docs/docs/project/creating-a-release.md
+++ b/docs/docs/project/creating-a-release.md
@@ -169,7 +169,7 @@ NEW_VERSION="${PAIMON_VERSION}" \
./tools/releasing/update_branch_version.sh
```
-Set `VERSION` in `paimon-python/setup.py` to the final
+Set `VERSION` in `paimon-python/pypaimon/_version.py` to the final
`PAIMON_VERSION`, without `.dev` or an RC suffix. The release workflow
derives the TestPyPI version by appending `rcRC_NUMBER`; the source candidate
keeps the final version.
@@ -180,9 +180,9 @@ candidate contains no snapshot or development version:
```shell
mvn -q -DforceStdout help:evaluate -Dexpression=project.version
-python3 paimon-python/setup.py --version
+python3 -c "import runpy;
print(runpy.run_path('paimon-python/pypaimon/_version.py')['VERSION'])"
rg --glob 'pom.xml' "<version>${PAIMON_VERSION}-SNAPSHOT</version>"
-rg '^VERSION = ".*\.dev' paimon-python/setup.py
+rg '^VERSION = ".*\.dev' paimon-python/pypaimon/_version.py
git status --short
```
@@ -316,7 +316,8 @@ RELEASE_VERSION="${PAIMON_VERSION}" \
./tools/releasing/create_source_release.sh
cd paimon-python
-python3 setup.py sdist
+python3 -m pip install --group build
+python3 -m build --sdist
gpg --armor --detach-sig \
"dist/pypaimon-${PAIMON_VERSION}.tar.gz"
diff --git a/docs/docs/project/releases.md b/docs/docs/project/releases.md
index 3e23b79fb7..5e72367d3f 100644
--- a/docs/docs/project/releases.md
+++ b/docs/docs/project/releases.md
@@ -65,7 +65,7 @@ PyPI package is built from the approved commit with the final
version.
## Release model
The combined Paimon and PyPaimon release uses one shared version number. The
-Maven project version and `paimon-python/setup.py` version must be equal.
+Maven project version and `paimon-python/pypaimon/_version.py` version must be
equal.
| Deliverable | Candidate | Published location |
| --- | --- | --- |
diff --git a/docs/docs/project/verifying-a-release-candidate.md
b/docs/docs/project/verifying-a-release-candidate.md
index f3cd609281..53c4d5455e 100644
--- a/docs/docs/project/verifying-a-release-candidate.md
+++ b/docs/docs/project/verifying-a-release-candidate.md
@@ -189,7 +189,7 @@ Check at least the following:
- All bundled third-party material is compatible with the Apache License 2.0
and is recorded in `LICENSE` or `NOTICE` where required.
- Maven POMs use `PAIMON_VERSION` without `-SNAPSHOT`.
-- `paimon-python/setup.py` and PyPaimon package metadata use
+- `paimon-python/pypaimon/_version.py` and PyPaimon package metadata use
`PAIMON_VERSION` without `.dev`.
Extract the Paimon candidate:
@@ -324,10 +324,11 @@ First build the official PyPaimon source candidate in an
isolated environment:
```shell
python3 -m venv pypaimon-rc-venv
. pypaimon-rc-venv/bin/activate
-python -m pip install --upgrade pip setuptools wheel
+python -m pip install --upgrade pip
cd "pypaimon-${PAIMON_VERSION}"
-python setup.py sdist bdist_wheel
+python -m pip install --group build
+python -m build
python -m pip install "dist/pypaimon-${PAIMON_VERSION}"*.whl
python -c "import pypaimon; print('PyPaimon import OK')"
cd ..
@@ -349,8 +350,7 @@ than assuming that a matrix from another release applies:
```shell
(
cd "paimon-${PAIMON_VERSION}/paimon-python"
- python -m pip install -r dev/requirements.txt
- python -m pip install -r dev/requirements-dev.txt
+ python -m pip install -e . --group dev
python -m pytest pypaimon/tests -v
)
```
@@ -358,6 +358,10 @@ than assuming that a matrix from another release applies:
Some optional integration tests need additional services or packages. Record
which tests ran, skipped, or failed.
+For Python 3.6/3.7, install a compatible `tomli`, run `python dev/install.py
legacy`,
+and use `bash dev/lint-python.sh -i pytest` to select the supported core test
subset.
+The complete dev group is intended for a modern Python environment.
+
## Verify the TestPyPI candidate
Install the exact RC version in a new environment:
diff --git a/docs/docs/pypaimon/installation.md
b/docs/docs/pypaimon/installation.md
index a4efd18b6b..d0a6647583 100644
--- a/docs/docs/pypaimon/installation.md
+++ b/docs/docs/pypaimon/installation.md
@@ -90,22 +90,24 @@ Python module with:
python -m pip install ./paimon-python
```
-For local Python development, use an editable installation:
+For local Python development, use pip with dependency-group support to install
+the editable project and development tools together:
```shell
-python -m pip install -e './paimon-python'
+cd paimon-python
+python -m pip install -e . --group dev
```
-To produce a source archive:
+To produce a source archive and wheel, run from `paimon-python/`:
```shell
-cd paimon-python
-python setup.py sdist
+python -m pip install --group build
+python -m build
```
-Archives are written to `paimon-python/dist/`. Development builds can produce
-both a base development archive and a dated archive; select one archive when
-installing instead of passing `dist/*.tar.gz` to pip.
+The source archive and wheel are written to `dist/` with the same version.
+Development versions append the Git commit date, for example `2.2.dev20260927`.
+Install the wheel with `python -m pip install dist/*.whl`.
## Verify the installation
diff --git a/paimon-python/MANIFEST.in b/paimon-python/MANIFEST.in
index d644332dce..35a33f1e5f 100644
--- a/paimon-python/MANIFEST.in
+++ b/paimon-python/MANIFEST.in
@@ -17,7 +17,7 @@
################################################################################
include LICENSE
include NOTICE
-include dev/requirements.txt
+include pyproject.toml
include README.md
recursive-include pypaimon *.py
recursive-include pypaimon/sample/data *.jsonl
diff --git a/paimon-python/README.md b/paimon-python/README.md
index 246af0693a..1b7c200b89 100644
--- a/paimon-python/README.md
+++ b/paimon-python/README.md
@@ -10,10 +10,25 @@ This PyPi package contains the Python APIs for using Paimon.
Pypaimon requires Python 3.6+.
-# Dependencies
+# Build
+
+Run from `paimon-python/` with pip supporting dependency groups.
+
+Normal build:
+
+```shell
+python -m pip install --group build
+python -m build
+```
-The core dependencies are listed in `dev/requirements.txt`.
-The development dependencies are listed in `dev/requirements-dev.txt`.
+Development build (editable installation with development dependencies):
+
+```shell
+python -m pip install -e . --group dev
+python -m build
+```
+
+Both produce a source archive and wheel in `dist/`.
# OSS metadata commits
@@ -89,22 +104,6 @@ All concurrent writers must use conditional creation. Older
Python clients or
other clients that overwrite snapshot objects can still overwrite a successful
commit. This change does not add conditional writes for other object stores.
-# Build
-
-You can build the source package by executing the following command:
-
-```commandline
-python3 setup.py sdist
-```
-
-The package is under `dist/`. Then you can install the package by executing
the following command:
-
-```commandline
-pip3 install dist/*.tar.gz
-```
-
-The command will install the package and core dependencies to your local
Python environment.
-
# Row ID column updates
For a batch update of selected columns in a data-evolution table with row
diff --git a/paimon-python/conftest.py b/paimon-python/conftest.py
index ef2e3b5ec1..7aa11d1fc6 100644
--- a/paimon-python/conftest.py
+++ b/paimon-python/conftest.py
@@ -198,7 +198,8 @@ def enable_native_backends(request, monkeypatch):
python_read = request.node.get_closest_marker("python_read") is not None
python_write = request.node.get_closest_marker("python_write") is not None
python_commit = request.node.get_closest_marker("python_commit") is not
None
- native_plan_test = request.path.name in (
+ # request.path requires pytest 7; Python 3.6 uses pytest 6.
+ native_plan_test = os.path.basename(request.node.location[0]) in (
"native_plan_test.py", "native_plan_integration_test.py",
"native_plan_capabilities_test.py")
force_plan = _native_plan_enabled() and not python_plan and not
native_plan_test
diff --git a/paimon-python/dev/check-licensing.sh
b/paimon-python/dev/check-licensing.sh
index c867a0d7b6..19bb7f013b 100755
--- a/paimon-python/dev/check-licensing.sh
+++ b/paimon-python/dev/check-licensing.sh
@@ -48,7 +48,7 @@ if [ -z "${SOURCE_PACKAGE}" ]; then
cd ${PROJECT_ROOT}
# Sanity check to ensure that resolved paths are valid; a LICENSE file
should always exist in project root
- if [ ! -f ${PROJECT_ROOT}/setup.py ]; then
+ if [ ! -f ${PROJECT_ROOT}/pyproject.toml ]; then
echo "Project root path ${PROJECT_ROOT} is not valid; script may be in
the wrong directory."
exit 1
fi
diff --git a/paimon-python/dev/install.py b/paimon-python/dev/install.py
new file mode 100644
index 0000000000..d1100606ce
--- /dev/null
+++ b/paimon-python/dev/install.py
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Install TOML dependency groups on old Python/pip without --group support.
+
+Modern development environments use ``pip install -e . --group dev`` directly.
+This bridge reads the same configuration for the Python 3.6/3.7 CI lanes.
+"""
+import argparse
+import os
+import re
+import subprocess
+import sys
+
+try:
+ import tomllib
+except ImportError:
+ import tomli as tomllib
+
+
+def _normalize(name):
+ return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def resolve_group(groups, name, parents=()):
+ name = _normalize(name)
+ if name in parents:
+ raise ValueError("Cyclic dependency group: {}".format(name))
+ requirements = []
+ for item in groups[name]:
+ if isinstance(item, str):
+ requirements.append(item)
+ elif isinstance(item, dict) and set(item) == {"include-group"}:
+ requirements.extend(resolve_group(
+ groups, item["include-group"], parents + (name,)))
+ else:
+ raise ValueError("Invalid dependency group item:
{!r}".format(item))
+ return requirements
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("group")
+ args = parser.parse_args()
+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ with open(os.path.join(root, "pyproject.toml"), "rb") as project_file:
+ raw_groups = tomllib.load(project_file)["dependency-groups"]
+ groups = {_normalize(name): value for name, value in raw_groups.items()}
+ if len(groups) != len(raw_groups):
+ raise ValueError("Duplicate normalized dependency group names")
+ subprocess.check_call(
+ [sys.executable, "-m", "pip", "install", root] +
+ resolve_group(groups, args.group))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/paimon-python/dev/lint-python.sh b/paimon-python/dev/lint-python.sh
index 1795379f17..86d77a6782 100755
--- a/paimon-python/dev/lint-python.sh
+++ b/paimon-python/dev/lint-python.sh
@@ -152,7 +152,7 @@ function license_check() {
# Flake8 check
function flake8_check() {
- local PRUNE_PATHS="\( -path ./dev -o -path ./.tox -o -path ./.venv"
+ local PRUNE_PATHS="\( -path ./dev -o -path ./.tox -o -path ./.venv -o
-path ./build -o -path ./dist"
if python -c "import sys; sys.exit(0 if sys.version_info < (3, 10) else
1)" 2>/dev/null; then
PRUNE_PATHS="$PRUNE_PATHS -o -path ./pypaimon/daft -o -path
./pypaimon/tests/daft"
fi
@@ -364,9 +364,5 @@ done
# collect checks according to the options
collect_checks
-if python -c "import sys; sys.exit(0 if sys.version_info >= (3, 8) else 1)";
then
- python -m pip install 'paimon-ftindex==0.1.0' || exit 1
-fi
-
# run checks
check_stage
diff --git a/paimon-python/dev/requirements-dev.txt
b/paimon-python/dev/requirements-dev.txt
deleted file mode 100644
index 711b0ee62a..0000000000
--- a/paimon-python/dev/requirements-dev.txt
+++ /dev/null
@@ -1,41 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-# Core dependencies for pypaimon are in requirements.txt
-
-# Test dependencies for pypaimon are as follows
-duckdb==1.3.2
-flake8==4.0.1
-oss2>=2.18,<3
-pytest~=7.0
-# HDF5 ingestion tests run in the supported h5py wheel lanes (Python 3.8+).
-h5py>=3,<4; python_version >= "3.8"
-# merge_into needs Dataset.join (added in Ray 2.50). Python 3.8 has no 2.50
wheel.
-ray>=2.10.0; python_version < "3.9"
-ray>=2.50.0; python_version >= "3.9"
-requests
-parameterized
-# Vortex 0.71.0 regresses native predicate pushdown on single-row files.
-vortex-data==0.70.0; python_version >= "3.11"
-# merge_into condition expressions (optional, for condition tests)
-datafusion>=52; python_version >= "3.10"
-# Lumina vector search (optional, for lumina index tests)
-lumina-data>=0.1.0
-# paimon-vindex vector search (optional, for vindex index tests)
-paimon-vindex==0.5.0; python_version >= "3.9"
-# paimon-ftindex full-text search (optional, for full-text index tests)
-paimon-ftindex==0.1.0; python_version >= "3.8"
diff --git a/paimon-python/dev/requirements.txt
b/paimon-python/dev/requirements.txt
deleted file mode 100644
index dc9b4b4e91..0000000000
--- a/paimon-python/dev/requirements.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-cachetools>=4.2,<6; python_version=="3.6"
-cachetools>=5,<6; python_version>"3.6"
-dataclasses>=0.8; python_version < "3.7"
-fastavro>=1.4,<2
-isal>=1.8,<2; python_version >= "3.9" and (platform_machine == "x86_64" or
platform_machine == "AMD64" or platform_machine == "aarch64" or
platform_machine == "arm64")
-fsspec>=2021.10,<2026; python_version<"3.8"
-fsspec>=2023,<2026; python_version>="3.8"
-packaging>=21,<26
-pandas>=1.1,<2; python_version < "3.7"
-pandas>=1.3,<3; python_version >= "3.7" and python_version < "3.9"
-pandas>=1.5,<3; python_version >= "3.9"
-polars>=0.9,<1; python_version<"3.8"
-polars>=1,<2; python_version=="3.8"
-polars>=1.32,<2; python_version>="3.9"
-pyarrow>=6,<7; python_version < "3.7"
-pyarrow>=7,<13; python_version >= "3.7" and python_version < "3.8"
-pyarrow>=16,!=19.0.0,<20; python_version >= "3.8"
-pyroaring<=0.3.3; python_version < "3.7"
-pyroaring<=0.4.5; python_version == "3.7"
-pyroaring>=1.0.0; python_version >= "3.8"
-readerwriterlock>=1,<2
-requests>=2.21.0,<3
-urllib3>=1.26,<3
-zstandard>=0.19,<1
-backports.zstd>=1.0.0,<1.4.0; python_version >= "3.9" and python_version <
"3.14"
-cramjam>=1.3.0,<3; python_version>="3.7"
-pyyaml>=5.4,<7
diff --git a/paimon-python/dev/run_mixed_tests.sh
b/paimon-python/dev/run_mixed_tests.sh
index fc84202a0d..84a54e21db 100755
--- a/paimon-python/dev/run_mixed_tests.sh
+++ b/paimon-python/dev/run_mixed_tests.sh
@@ -660,29 +660,8 @@ ensure_paimon_vindex() {
return 0
fi
- echo "Installing Python paimon-vindex dependency..."
- if python -m pip install 'paimon-vindex==0.5.0'; then
- return 0
- fi
-
- echo -e "${YELLOW}Direct pip install failed; installing paimon-vindex into
a temporary target directory...${NC}"
- local target_dir="${TMPDIR:-/tmp}/paimon-vindex-site"
- rm -rf "$target_dir"
- if python -m pip install --target "$target_dir" 'paimon-vindex==0.5.0';
then
- export PYTHONPATH="$target_dir:${PYTHONPATH:-}"
- return 0
- fi
-
- if python -c "import numpy" >/dev/null 2>&1; then
- echo -e "${YELLOW}Dependency install failed but numpy is already
available; retrying paimon-vindex without dependencies...${NC}"
- rm -rf "$target_dir"
- if python -m pip install --target "$target_dir" --no-deps
'paimon-vindex==0.5.0'; then
- export PYTHONPATH="$target_dir:${PYTHONPATH:-}"
- return 0
- fi
- fi
-
- echo -e "${RED}✗ Failed to install paimon-vindex${NC}"
+ echo -e "${RED}Missing paimon-vindex; install the Python dev dependency
group first.${NC}"
+ echo "From paimon-python: python -m pip install -e . --group dev"
return 1
}
diff --git a/paimon-python/dev/test_packaging.py
b/paimon-python/dev/test_packaging.py
new file mode 100644
index 0000000000..e9d0e5ea52
--- /dev/null
+++ b/paimon-python/dev/test_packaging.py
@@ -0,0 +1,212 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Exercise actual distributions without importing runtime/native dependencies.
+
+Run with ``python dev/test_packaging.py`` after installing the build tools.
+The same suite runs on Python 3.6 and modern Python in CI.
+"""
+import email
+import os
+from pathlib import Path
+import runpy
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+import unittest
+import zipfile
+
+from packaging.markers import default_environment
+from packaging.requirements import Requirement
+from packaging.utils import canonicalize_name
+from packaging.version import Version
+
+try:
+ import tomllib
+except ImportError:
+ import tomli as tomllib
+
+
+def command(args, cwd, env=None):
+ result = subprocess.run(args, cwd=str(cwd), env=env,
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ if result.returncode:
+ raise AssertionError(result.stdout.decode("utf-8",
errors="replace")[-8000:])
+ return result.stdout.decode("utf-8")
+
+
+def wheel_contents(path):
+ with zipfile.ZipFile(str(path)) as archive:
+ names = archive.namelist()
+ metadata = email.message_from_bytes(archive.read(next(
+ name for name in names if name.endswith(".dist-info/METADATA"))))
+ full_version = archive.read("pypaimon/_full_version").decode().strip()
+ return names, metadata, full_version
+
+
+class PackagingTest(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.temp = tempfile.TemporaryDirectory(prefix="pypaimon-packaging-")
+ cls.root = Path(cls.temp.name)
+ cls.source = cls.root / "source"
+ project = Path(__file__).resolve().parent.parent
+ shutil.copytree(str(project), str(cls.source),
ignore=shutil.ignore_patterns(
+ "build", "dist", "*.egg-info", "__pycache__", ".pytest_cache",
".venv"))
+ cls.env = os.environ.copy()
+ for key in ("GIT_DIR", "GIT_WORK_TREE", "PYPAIMON_BUILD_VERSION"):
+ cls.env.pop(key, None)
+ cls.env.update(GIT_AUTHOR_DATE="2020-01-02T03:04:05+00:00",
+ GIT_COMMITTER_DATE="2020-01-02T03:04:05+00:00")
+ cls.commit = cls.init_git(cls.source)
+ base = runpy.run_path(str(cls.source /
"pypaimon/_version.py"))["VERSION"]
+ cls.version = str(Version(base + "20200102" if base.endswith(".dev")
else base))
+ with (cls.source / "pyproject.toml").open("rb") as config:
+ cls.config = tomllib.load(config)
+ cls.dist = cls.root / "custom-output"
+ cls.build(cls.source, cls.dist)
+ cls.wheel = next(cls.dist.glob("*.whl"))
+ cls.sdist = next(cls.dist.glob("*.tar.gz"))
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.temp.cleanup()
+
+ @classmethod
+ def init_git(cls, root):
+ for args in (["init", "-q"], ["config", "user.name", "Packaging Test"],
+ ["config", "user.email", "[email protected]"], ["add",
"."],
+ ["commit", "-qm", "test source"]):
+ command(["git"] + args, root, cls.env)
+ return command(["git", "rev-parse", "HEAD"], root, cls.env).strip()
+
+ @classmethod
+ def build(cls, source, output, extra=(), env=None):
+ command([sys.executable, "-m", "build", "--no-isolation", "--outdir",
+ str(output)] + list(extra), source, env or cls.env)
+
+ def test_sdist_and_wheel_share_version_and_commit(self):
+ self.assertEqual(2, len(list(self.dist.iterdir())))
+ self.assertEqual("pypaimon-{}.tar.gz".format(self.version),
self.sdist.name)
+ self.assertEqual("pypaimon-{}-py3-none-any.whl".format(self.version),
self.wheel.name)
+ _, metadata, full_version = wheel_contents(self.wheel)
+ self.assertEqual(self.version, metadata["Version"])
+ self.assertEqual("python-{}-{}".format(self.version, self.commit),
full_version)
+ with tarfile.open(str(self.sdist)) as archive:
+ prefix = "pypaimon-{}/".format(self.version)
+ self.assertEqual(full_version, archive.extractfile(
+ prefix + "pypaimon/_full_version").read().decode().strip())
+
+ def test_distribution_contents(self):
+ names, metadata, _ = wheel_contents(self.wheel)
+ self.assertEqual(">=3.6", metadata["Requires-Python"])
+ self.assertFalse(any(name.startswith(("pypaimon/tests/",
"pypaimon/acceptance/"))
+ for name in names))
+ self.assertFalse(any(name.endswith((".so", ".pyd", ".jar")) for name
in names))
+ for name in ("pypaimon/sample/data/data.jsonl", "pypaimon/_version.py",
+ "pypaimon/benchmark/act/default_experiment.json"):
+ self.assertIn(name, names)
+ for suffix in ("/LICENSE", "/NOTICE"):
+ self.assertTrue(any(".dist-info/" in name and name.endswith(suffix)
+ for name in names))
+ with tarfile.open(str(self.sdist)) as archive:
+ names = archive.getnames()
+ prefix = "pypaimon-{}/".format(self.version)
+ for name in ("pyproject.toml", "setup.py", "LICENSE", "NOTICE",
"README.md"):
+ self.assertIn(prefix + name, names)
+ self.assertFalse(any("requirements.txt" in name or
"requirements-dev.txt" in name
+ or name.startswith((prefix +
"pypaimon/tests/",
+ prefix +
"pypaimon/acceptance/"))
+ for name in names))
+
+ @staticmethod
+ def active_requirements(values, environment):
+ result = set()
+ for value in values:
+ requirement = Requirement(value)
+ if requirement.marker is None or
requirement.marker.evaluate(environment):
+ result.add((canonicalize_name(requirement.name),
str(requirement.specifier)))
+ return result
+
+ def test_metadata_preserves_core_and_extra_conditions(self):
+ _, metadata, _ = wheel_contents(self.wheel)
+ project = self.config["project"]
+ self.assertEqual(set(project["optional-dependencies"]),
+ set(metadata.get_all("Provides-Extra")))
+ for python_version in ("3.6", "3.7", "3.8", "3.11", "3.13"):
+ for system in ("Linux", "Windows"):
+ for extra in [""] + list(project["optional-dependencies"]):
+ with self.subTest(python=python_version, system=system,
extra=extra):
+ environment = default_environment()
+ environment.update(python_version=python_version,
+ python_full_version=python_version
+ ".0",
+ platform_system=system, extra=extra)
+ expected = project["dependencies"] + project[
+ "optional-dependencies"].get(extra, [])
+ self.assertEqual(self.active_requirements(expected,
environment),
+ self.active_requirements(
+
metadata.get_all("Requires-Dist"), environment))
+
+ def test_dev_group_reuses_product_extras(self):
+ resolve = runpy.run_path(str(self.source /
"dev/install.py"))["resolve_group"]
+ requirements = [Requirement(value) for value in resolve(
+ self.config["dependency-groups"], "dev")]
+ project = next(requirement for requirement in requirements if
requirement.name == "pypaimon")
+ self.assertTrue({"ray", "sql", "vindex",
"full-text"}.issubset(project.extras))
+ # Product constraints belong to extras, not a second developer copy.
+ names = {requirement.name for requirement in requirements}
+ self.assertFalse(names.intersection({"datafusion", "paimon-vindex",
"requests", "h5py"}))
+
+ def test_sdist_rebuild_preserves_provenance_in_another_git_repo(self):
+ outer = self.root / "downstream"
+ outer.mkdir()
+ (outer / "README").write_text("Unrelated downstream repository")
+ downstream_commit = self.init_git(outer)
+ self.assertNotEqual(self.commit, downstream_commit)
+ with tarfile.open(str(self.sdist)) as archive:
+ # This archive was just built from our test source.
+ archive.extractall(str(outer))
+ output = self.root / "rebuilt"
+ self.build(outer / "pypaimon-{}".format(self.version), output,
extra=("--wheel",))
+ _, metadata, full_version = wheel_contents(next(output.glob("*.whl")))
+ self.assertEqual(self.version, metadata["Version"])
+ self.assertEqual("python-{}-{}".format(self.version, self.commit),
full_version)
+
+ def test_release_override_applies_to_both_formats(self):
+ env = dict(self.env, PYPAIMON_BUILD_VERSION="2.2rc1")
+ output = self.root / "release-candidate"
+ self.build(self.source, output, env=env)
+ self.assertEqual({"pypaimon-2.2rc1.tar.gz",
"pypaimon-2.2rc1-py3-none-any.whl"},
+ {path.name for path in output.iterdir()})
+ _, metadata, full_version = wheel_contents(next(output.glob("*.whl")))
+ self.assertEqual("2.2rc1", metadata["Version"])
+ self.assertEqual("python-2.2rc1-{}".format(self.commit), full_version)
+ self.assertEqual("", command(["git", "diff", "--",
"pypaimon/_version.py"],
+ self.source, self.env))
+
+ def test_invalid_release_override_fails(self):
+ env = dict(self.env, PYPAIMON_BUILD_VERSION="not a version")
+ result = subprocess.run([sys.executable, "-m", "build",
"--no-isolation", "--sdist"],
+ cwd=str(self.source), env=env,
+ stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
+ self.assertNotEqual(0, result.returncode)
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/paimon-python/MANIFEST.in b/paimon-python/pypaimon/_version.py
similarity index 77%
copy from paimon-python/MANIFEST.in
copy to paimon-python/pypaimon/_version.py
index d644332dce..7948185ec7 100644
--- a/paimon-python/MANIFEST.in
+++ b/paimon-python/pypaimon/_version.py
@@ -1,4 +1,4 @@
-################################################################################
+##########################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -14,12 +14,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-################################################################################
-include LICENSE
-include NOTICE
-include dev/requirements.txt
-include README.md
-recursive-include pypaimon *.py
-recursive-include pypaimon/sample/data *.jsonl
-recursive-exclude pypaimon/tests *
-recursive-exclude pypaimon/acceptance *
+##########################################################################
+# Base version; development distributions append the Git commit date.
+VERSION = "2.2.dev"
diff --git a/paimon-python/pypaimon/build_info.py
b/paimon-python/pypaimon/build_info.py
index 22649caf77..6e8cd4cc41 100644
--- a/paimon-python/pypaimon/build_info.py
+++ b/paimon-python/pypaimon/build_info.py
@@ -16,12 +16,12 @@
# under the License.
import os
-import re
+import runpy
import subprocess
_UNKNOWN = "UNKNOWN"
_FULL_VERSION_FILE = os.path.join(os.path.dirname(__file__), "_full_version")
-_SETUP_FILE = os.path.join(os.path.dirname(os.path.dirname(__file__)),
"setup.py")
+_VERSION_FILE = os.path.join(os.path.dirname(__file__), "_version.py")
def _repository_root():
@@ -33,36 +33,33 @@ def _repository_root():
return python_root
-def git_commit_id():
- """Return the current Paimon Git revision without discovering an outer
repo."""
+def _git_output(args):
repository_root = _repository_root()
env = os.environ.copy()
env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(repository_root)
try:
return subprocess.check_output(
- ["git", "-C", repository_root, "rev-parse", "HEAD"],
+ ["git", "-C", repository_root] + args,
stderr=subprocess.DEVNULL,
env=env,
).decode("utf-8").strip()
except Exception:
- return _UNKNOWN
+ return None
+
+
+def git_commit_id():
+ """Return the current Paimon Git revision without discovering an outer
repo."""
+ return _git_output(["rev-parse", "HEAD"]) or _UNKNOWN
def _source_version():
try:
- with open(_SETUP_FILE, "r") as setup_file:
- match = re.search(
- r'^VERSION = ["\']([^"\']+)["\']$',
- setup_file.read(),
- re.MULTILINE,
- )
- return None if match is None else match.group(1)
+ return runpy.run_path(_VERSION_FILE)["VERSION"]
except OSError:
return None
-def _load_full_version():
- """Return the embedded full version, or derive it from the checkout."""
+def _embedded_full_version():
try:
with open(_FULL_VERSION_FILE, "r") as full_version_file:
value = full_version_file.read().strip()
@@ -70,7 +67,28 @@ def _load_full_version():
return value
except OSError:
pass
+ return None
+
+
+def package_version():
+ """Resolve one version for metadata, sdist and wheel, even outside Git."""
+ embedded = _embedded_full_version()
+ if embedded:
+ return embedded[len("python-"):].rsplit("-", 1)[0]
version = _source_version()
+ if version and version.endswith(".dev"):
+ # Commit date, rather than build date, keeps source rebuilds stable.
+ date = _git_output(["log", "-1", "--format=%cd",
"--date=format:%Y%m%d"])
+ return version + (date or "0")
+ return version
+
+
+def _load_full_version():
+ """Return the embedded full version, or derive it from the checkout."""
+ embedded = _embedded_full_version()
+ if embedded:
+ return embedded
+ version = package_version()
return (
_UNKNOWN
if version is None
diff --git a/paimon-python/pypaimon/tests/build_info_test.py
b/paimon-python/pypaimon/tests/build_info_test.py
index a3dc11627e..b7a08aea69 100644
--- a/paimon-python/pypaimon/tests/build_info_test.py
+++ b/paimon-python/pypaimon/tests/build_info_test.py
@@ -34,7 +34,7 @@ class BuildInfoTest(unittest.TestCase):
self.assertRegex(
build_info.full_version(),
r"^python-{}-(UNKNOWN|[0-9a-f]{{40}})$".format(
- re.escape(build_info._source_version())),
+ re.escape(build_info.package_version())),
)
def test_embedded_full_version(self):
@@ -106,9 +106,10 @@ class BuildInfoTest(unittest.TestCase):
embedded_file = os.path.join(extracted, "pypaimon",
"_full_version")
with open(embedded_file, "r") as full_version_file:
embedded = full_version_file.read().strip()
+ self.assertTrue(embedded.endswith("-" + upstream_commit))
self.assertEqual(
- "python-{}-{}".format(build_info._source_version(),
upstream_commit),
- embedded,
+ "pypaimon-{}".format(embedded[len("python-"):].rsplit("-",
1)[0]),
+ top_level,
)
downstream_commit = self._init_git_repository(extracted,
"downstream")
diff --git a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
index f43c354b6e..f30bd443ca 100644
--- a/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
+++ b/paimon-python/pypaimon/tests/ray_data_evolution_merge_into_test.py
@@ -68,7 +68,8 @@ class RayDataEvolutionMergeIntoTest(unittest.TestCase):
cls.catalog = CatalogFactory.create(cls.catalog_options)
cls.catalog.create_database('default', True)
if not ray.is_initialized():
- ray.init(ignore_reinit_error=True, num_cpus=2)
+ # Nested joins and the final shuffle retain actors while
scheduling tasks.
+ ray.init(ignore_reinit_error=True, num_cpus=4)
@classmethod
def tearDownClass(cls):
diff --git a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
index 4c9a9fd066..eb5d4b4594 100644
--- a/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
+++ b/paimon-python/pypaimon/tests/rest/rest_read_write_test.py
@@ -546,10 +546,8 @@ class RESTTableReadWriteTest(RESTBaseTest):
self.assertIsNotNone(ray_dataset, "Ray dataset should not be None")
self.assertEqual(ray_dataset.count(), 8, "Should have 8 rows")
- df = ray_dataset.to_pandas()
- expect = pd.DataFrame(self.raw_data)
-
pd.testing.assert_frame_equal(df.sort_values(by='user_id').reset_index(drop=True),
-
expect.sort_values(by='user_id').reset_index(drop=True))
+ actual = pa.concat_tables(ray.get(ray_dataset.to_arrow_refs()))
+ self.assertEqual(actual.sort_by('user_id'),
self.expected.sort_by('user_id'))
def test_ray_data_write_and_read(self):
if not ray.is_initialized():
@@ -587,15 +585,8 @@ class RESTTableReadWriteTest(RESTBaseTest):
self.assertIsNotNone(ray_dataset, "Ray dataset should not be None")
self.assertEqual(ray_dataset.count(), 3, "Should have 3 rows")
- df = ray_dataset.to_pandas()
- expected_df = pd.DataFrame({
- 'id': [1, 2, 3],
- 'name': ['Alice', 'Bob', 'Charlie'],
- 'value': [100, 200, 300],
- })
- expected_df['id'] = expected_df['id'].astype('int32')
-
pd.testing.assert_frame_equal(df.sort_values(by='id').reset_index(drop=True),
-
expected_df.sort_values(by='id').reset_index(drop=True))
+ actual = pa.concat_tables(ray.get(ray_dataset.to_arrow_refs()))
+ self.assertEqual(actual.sort_by('id'), test_data.sort_by('id'))
def test_write_wide_table_large_data(self):
logging.basicConfig(level=logging.INFO)
diff --git a/paimon-python/pyproject.toml b/paimon-python/pyproject.toml
new file mode 100644
index 0000000000..d39be4f839
--- /dev/null
+++ b/paimon-python/pyproject.toml
@@ -0,0 +1,211 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+[build-system]
+requires = [
+ "setuptools>=40.8,<60; python_version < '3.7'",
+ "setuptools>=61; python_version >= '3.7'",
+ "wheel",
+ "packaging>=21",
+ "tomli>=1.2.3; python_version < '3.11'",
+]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pypaimon"
+dynamic = ["version"]
+description = "Apache Paimon Python API"
+readme = "README.md"
+# Keep the table form for setuptools versions supporting Python 3.7/3.8.
+license = {text = "Apache-2.0"}
+authors = [{name = "Apache Software Foundation", email =
"[email protected]"}]
+requires-python = ">=3.6"
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Programming Language :: Python :: 3.6",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Programming Language :: Python :: 3.13",
+]
+dependencies = [
+ "cachetools>=4.2,<6; python_version == '3.6'",
+ "cachetools>=5,<6; python_version > '3.6'",
+ "dataclasses>=0.8; python_version < '3.7'",
+ "fastavro>=1.4,<2",
+ "isal>=1.8,<2; python_version >= '3.9' and (platform_machine == 'x86_64'
or platform_machine == 'AMD64' or platform_machine == 'aarch64' or
platform_machine == 'arm64')",
+ "fsspec>=2021.10,<2026; python_version < '3.8'",
+ "fsspec>=2023,<2026; python_version >= '3.8'",
+ "packaging>=21",
+ "pandas>=1.1,<2; python_version < '3.7'",
+ "pandas>=1.3,<3; python_version >= '3.7' and python_version < '3.9'",
+ "pandas>=1.5,<3; python_version >= '3.9'",
+ "polars>=0.9,<1; python_version < '3.8'",
+ "polars>=1,<2; python_version == '3.8'",
+ "polars>=1.32,<2; python_version >= '3.9'",
+ "pyarrow>=6,<7; python_version < '3.7'",
+ "pyarrow>=7,<13; python_version >= '3.7' and python_version < '3.8'",
+ "pyarrow>=16,!=19.0.0,<20; python_version >= '3.8'",
+ "pyroaring<=0.3.3; python_version < '3.7'",
+ "pyroaring<=0.4.5; python_version == '3.7'",
+ "pyroaring>=1.0.0; python_version >= '3.8'",
+ "readerwriterlock>=1,<2",
+ "requests>=2.21.0,<3",
+ "urllib3>=1.26,<3",
+ "zstandard>=0.19,<1",
+ "backports.zstd>=1.0.0,<1.4.0; python_version >= '3.9' and python_version
< '3.14'",
+ "cramjam>=1.3.0,<3; python_version >= '3.7'",
+ "pyyaml>=5.4,<7",
+]
+
+[project.urls]
+Homepage = "https://paimon.apache.org";
+Documentation = "https://paimon.apache.org/docs/master/pypaimon/python-api/";
+
+[project.scripts]
+paimon = "pypaimon.cli:main"
+
+[project.optional-dependencies]
+video = [
+ "av>=12,<19; python_version >= '3.10'",
+ "Pillow; python_version >= '3.10'",
+]
+hdf5 = [
+ "h5py>=3,<4; python_version >= '3.8'",
+]
+rosbag = [
+ "rosbags>=0.11.5,<0.12; python_version >= '3.10'",
+]
+# datasets 4.1+ may require PyArrow 21+, outside the supported core range.
+# Pandas 2.2.2+ supports NumPy 2 selected by the media dependencies.
+lerobot = [
+ "datasets>=4,<4.1; python_version >= '3.10'",
+ "pandas>=2.2.2,<3; python_version >= '3.10'",
+ "torch>=2.3; python_version >= '3.10'",
+ "lerobot>=0.4.4,<0.5; python_version >= '3.10'",
+]
+ray = [
+ "ray>=2.10,<3; python_version >= '3.8'",
+]
+torch = [
+ "torch",
+ "Pillow",
+]
+# ACT uses the same LeRobot stack; do not maintain its versions twice.
+act = [
+ "pypaimon[lerobot]; python_version >= '3.10'",
+ "Pillow; python_version >= '3.10'",
+]
+daft = [
+ "daft>=0.7.6; python_version >= '3.10'",
+]
+oss = [
+ "oss2>=2.18,<3",
+ "ossfs>=2021.8; python_version < '3.8'",
+ "ossfs>=2023; python_version >= '3.8'",
+]
+jindo = [
+ "oss2>=2.18,<3",
+ "pyjindosdk>=6.10.4",
+]
+lance = [
+ "pylance>=0.20,<1; python_version >= '3.9'",
+ "pylance>=0.10,<1; python_version >= '3.8' and python_version < '3.9'",
+]
+# Vortex 0.71 regresses native predicate pushdown on single-row files.
+vortex = [
+ "vortex-data==0.70.0; python_version >= '3.11'",
+]
+mosaic = [
+ "paimon-mosaic>=0.1.0; python_version >= '3.9'",
+]
+lumina = [
+ "lumina-data>=0.1.0",
+]
+vindex = [
+ "paimon-vindex==0.5.0; python_version >= '3.9'",
+]
+full-text = [
+ "paimon-ftindex==0.1.0; python_version >= '3.8'",
+]
+theta-sketch = [
+ "datasketches>=4,<5; python_version < '3.9'",
+ "datasketches>=5,<6; python_version >= '3.9'",
+]
+hll-sketch = [
+ "datasketches>=4,<5; python_version < '3.9'",
+ "datasketches>=5,<6; python_version >= '3.9'",
+]
+datafusion = [
+ "datafusion>=54,<55; python_version >= '3.10'",
+]
+sql = [
+ "pypaimon[datafusion]; python_version >= '3.10'",
+ "pypaimon-rust>=0.3.0; python_version >= '3.10'",
+]
+hdfs = [
+ "hdfs-native>=0.13,<1; python_version >= '3.10' and platform_system !=
'Windows'",
+]
+
+[dependency-groups]
+# Tools are separated from optional integrations so old-Python core tests and
+# targeted compatibility jobs do not install unrelated native dependencies.
+test = [
+ "pytest>=6.2,<7; python_version < '3.7'",
+ "pytest>=7,<8; python_version >= '3.7'",
+ "parameterized>=0.8,<0.9; python_version < '3.7'",
+ "parameterized>=0.9,<1; python_version >= '3.7'",
+ "py4j==0.10.9.9",
+]
+lint = [
+ "flake8>=4,<5; python_version < '3.8'",
+ "flake8>=7,<8; python_version >= '3.8'",
+]
+dev = [
+ {include-group = "test"},
+ {include-group = "lint"},
+ {include-group = "build"},
+
"pypaimon[oss,lance,ray,daft,sql,hdf5,rosbag,video,vortex,mosaic,lumina,vindex,full-text,theta-sketch,hll-sketch]",
+ "duckdb>=1.3,<1.4; python_version >= '3.9'",
+ # merge_into / row-id operations require Dataset.join; basic Ray I/O does
not.
+ "ray>=2.50; python_version >= '3.9'",
+]
+torch = [
+ {include-group = "dev"},
+ "pypaimon[torch,lerobot]",
+]
+legacy = [
+ {include-group = "test"},
+ {include-group = "lint"},
+ "pypaimon[oss,lumina,theta-sketch,hll-sketch]",
+]
+build = ["build", "twine"]
+
+[tool.setuptools]
+include-package-data = true
+license-files = ["LICENSE", "NOTICE"]
+
+[tool.setuptools.packages.find]
+include = ["pypaimon*"]
+exclude = ["pypaimon.tests*", "pypaimon.acceptance*"]
+
+[tool.setuptools.package-data]
+pypaimon = ["_full_version", "benchmark/act/default_experiment.json"]
diff --git a/paimon-python/setup.py b/paimon-python/setup.py
index 47e600343e..e673fe50bf 100644
--- a/paimon-python/setup.py
+++ b/paimon-python/setup.py
@@ -15,68 +15,29 @@
# See the License for the specific language governing permissions and
# limitations under the License.
##########################################################################
-import atexit
+"""Setuptools hooks and the Python 3.6 bridge for pyproject.toml metadata."""
import os
-import shutil
-import subprocess
+import runpy
import sys
-import tarfile
-import tempfile
+from packaging.version import Version
from setuptools import find_packages, setup
from setuptools.command.build_py import build_py
from setuptools.command.sdist import sdist
PYTHON_ROOT = os.path.dirname(os.path.abspath(__file__))
-FULL_VERSION_FILE = os.path.join(PYTHON_ROOT, "pypaimon", "_full_version")
-UNKNOWN_COMMIT_ID = "UNKNOWN"
-
-VERSION = "2.2.dev"
-
-
-def _repository_root():
- parent = os.path.dirname(PYTHON_ROOT)
- if os.path.basename(PYTHON_ROOT) == "paimon-python" and os.path.exists(
- os.path.join(parent, "pom.xml")):
- return parent
- return PYTHON_ROOT
-
-
-def _git_output(args):
- repository_root = _repository_root()
- env = os.environ.copy()
- env["GIT_CEILING_DIRECTORIES"] = os.path.dirname(repository_root)
- try:
- return subprocess.check_output(
- ["git", "-C", repository_root] + args,
- stderr=subprocess.DEVNULL,
- env=env,
- ).decode("utf-8").strip()
- except Exception:
- return None
-
-
-def _full_version():
- try:
- with open(FULL_VERSION_FILE, "r") as full_version_file:
- embedded = full_version_file.read().strip()
- if embedded:
- return embedded
- except OSError:
- pass
-
- git_commit_id = _git_output(["rev-parse", "HEAD"])
- if git_commit_id is None:
- git_commit_id = UNKNOWN_COMMIT_ID
- return "python-{}-{}".format(VERSION, git_commit_id)
+BUILD_INFO = runpy.run_path(os.path.join(PYTHON_ROOT, "pypaimon",
"build_info.py"))
+VERSION = str(Version(os.environ.get("PYPAIMON_BUILD_VERSION") or
+ BUILD_INFO["package_version"]()))
+COMMIT_ID = BUILD_INFO["full_version"]().rsplit("-", 1)[-1]
+FULL_VERSION = "python-{}-{}".format(VERSION, COMMIT_ID)
def _write_full_version(root):
package_dir = os.path.join(root, "pypaimon")
- if not os.path.exists(package_dir):
- os.makedirs(package_dir)
- with open(os.path.join(package_dir, "_full_version"), "w") as
full_version_file:
- full_version_file.write(_full_version() + "\n")
+ os.makedirs(package_dir, exist_ok=True)
+ with open(os.path.join(package_dir, "_full_version"), "w") as version_file:
+ version_file.write(FULL_VERSION + "\n")
class PaimonBuildPy(build_py):
@@ -93,254 +54,44 @@ class PaimonSdist(sdist):
_write_full_version(base_dir)
-def get_dev_version():
- """Generate dev version with commit date.
- Format: 2.2.devYYYYMMDD (e.g. 2.2.dev20260911)
- Uses the commit date (author date) for reproducibility.
- """
- base = VERSION.rstrip(".")
- if not base.endswith("dev"):
- return None
-
- try:
- date_str = _git_output(
- ["log", "-1", "--format=%cd", "--date=format:%Y%m%d"])
- if date_str is None:
- raise RuntimeError("Git commit date is unavailable")
- except Exception:
- print("Warning: git not available, skipping dev package.")
- return None
-
- return base + date_str
-
-
-def _build_dev_package():
- """After sdist completes, repack a copy with dev version."""
- if "sdist" not in sys.argv:
- return
-
- dev_version = get_dev_version()
- if dev_version is None:
- return
-
- from packaging.version import Version
- normalized = str(Version(VERSION))
- dev_normalized = str(Version(dev_version))
-
- src_name = "pypaimon-{}".format(normalized)
- dev_name = "pypaimon-{}".format(dev_normalized)
-
- src_tar = os.path.join("dist", src_name + ".tar.gz")
- if not os.path.exists(src_tar):
- return
-
- tmp_dir = tempfile.mkdtemp()
- try:
- with tarfile.open(src_tar, "r:gz") as tar:
- tar.extractall(tmp_dir)
-
- src_dir = os.path.join(tmp_dir, src_name)
- dev_dir = os.path.join(tmp_dir, dev_name)
- os.rename(src_dir, dev_dir)
-
- # Update version in PKG-INFO files
- for pkg_info in [
- os.path.join(dev_dir, "PKG-INFO"),
- os.path.join(dev_dir, "pypaimon.egg-info", "PKG-INFO"),
- ]:
- if os.path.exists(pkg_info):
- with open(pkg_info, "r") as f:
- content = f.read()
- content = content.replace(
- "Version: " + normalized,
- "Version: " + dev_normalized
- )
- with open(pkg_info, "w") as f:
- f.write(content)
-
- # Update VERSION in setup.py so pip install gets the correct version
- setup_py = os.path.join(dev_dir, "setup.py")
- if os.path.exists(setup_py):
- with open(setup_py, "r") as f:
- content = f.read()
- content = content.replace(
- 'VERSION = "' + VERSION + '"',
- 'VERSION = "' + dev_version + '"'
- )
- with open(setup_py, "w") as f:
- f.write(content)
-
- # Keep the embedded full version consistent with the dev package
version.
- full_version_file = os.path.join(dev_dir, "pypaimon", "_full_version")
- if os.path.exists(full_version_file):
- with open(full_version_file, "r") as f:
- content = f.read()
- version_prefix = "python-{}-".format(VERSION)
- if content.startswith(version_prefix):
- content = "python-{}-{}".format(
- dev_version, content[len(version_prefix):])
- with open(full_version_file, "w") as f:
- f.write(content)
-
- dev_tar = os.path.join("dist", dev_name + ".tar.gz")
- with tarfile.open(dev_tar, "w:gz") as tar:
- tar.add(dev_dir, arcname=dev_name)
-
- print("Created dev package: " + dev_tar)
- finally:
- shutil.rmtree(tmp_dir)
-
-
-atexit.register(_build_dev_package)
-
-PACKAGES = find_packages(
- include=["pypaimon*"],
- exclude=["pypaimon.tests*", "pypaimon.acceptance*"],
-)
-
-
-def read_requirements():
- """Read requirements from dev/requirements.txt file."""
- requirements_path = os.path.join(os.path.dirname(__file__), 'dev',
'requirements.txt')
- requirements = []
-
- if os.path.exists(requirements_path):
- with open(requirements_path, 'r', encoding='utf-8') as f:
- for line in f:
- line = line.strip()
- # Skip empty lines and comments
- if line and not line.startswith('#'):
- requirements.append(line)
-
- return requirements
-
-
-install_requires = read_requirements()
-
-VIDEO_DEPENDENCIES = [
- 'av>=12,<19; python_version>="3.10"',
- 'Pillow; python_version>="3.10"',
-]
-
-LEROBOT_DEPENDENCIES = [
- # datasets 4.1+ may select PyArrow 21+, while PyPaimon currently
- # supports PyArrow <20. Pandas 2.2.2+ supports NumPy 2.x selected
- # by LeRobot's media dependencies.
- 'datasets>=4,<4.1; python_version>="3.10"',
- 'pandas>=2.2.2,<3; python_version>="3.10"',
- 'torch>=2.3; python_version>="3.10"',
- 'lerobot>=0.4.4,<0.5; python_version>="3.10"',
-]
+def _legacy_metadata():
+ # Python 3.6's last setuptools does not support PEP 621. Translate the same
+ # TOML metadata instead of keeping another dependency list or dropping 3.6.
+ import tomli
+
+ with open(os.path.join(PYTHON_ROOT, "pyproject.toml"), "rb") as
project_file:
+ config = tomli.load(project_file)
+ project = config["project"]
+ setuptools_config = config["tool"]["setuptools"]
+ with open(os.path.join(PYTHON_ROOT, project["readme"]), encoding="utf-8")
as readme:
+ long_description = readme.read()
+ author = project["authors"][0]
+ return dict(
+ name=project["name"],
+ description=project["description"],
+ long_description=long_description,
+ long_description_content_type="text/markdown",
+ license=project["license"]["text"],
+ author=author["name"],
+ author_email=author["email"],
+ url=project["urls"]["Homepage"],
+ project_urls=project["urls"],
+ python_requires=project["requires-python"],
+ classifiers=project["classifiers"],
+ install_requires=project["dependencies"],
+ extras_require=project["optional-dependencies"],
+ entry_points={"console_scripts": [
+ "{}={}".format(name, entry) for name, entry in
project["scripts"].items()
+ ]},
+ packages=find_packages(**setuptools_config["packages"]["find"]),
+ include_package_data=setuptools_config["include-package-data"],
+ package_data=setuptools_config["package-data"],
+ license_files=setuptools_config["license-files"],
+ )
-long_description = "See Apache Paimon Python API \
-[Doc](https://paimon.apache.org/docs/master/pypaimon/python-api/) for usage."
setup(
- name="pypaimon",
version=VERSION,
- packages=PACKAGES,
- include_package_data=True,
- package_data={
- "pypaimon": [
- "_full_version",
- "benchmark/act/default_experiment.json",
- ],
- },
cmdclass={"build_py": PaimonBuildPy, "sdist": PaimonSdist},
- install_requires=install_requires,
- entry_points={
- 'console_scripts': [
- 'paimon=pypaimon.cli:main',
- ],
- },
- extras_require={
- 'video': VIDEO_DEPENDENCIES,
- 'hdf5': [
- # HDF5 loading is explicitly guarded and documented as Python 3.8+.
- 'h5py>=3,<4; python_version>="3.8"',
- ],
- 'rosbag': [
- # rosbags is pure Python and does not require a ROS installation.
- 'rosbags>=0.11.5,<0.12; python_version>="3.10"',
- ],
- 'lerobot': LEROBOT_DEPENDENCIES,
- 'ray': [
- 'ray>=2.10,<3; python_version>="3.8"',
- ],
- 'torch': [
- 'torch',
- 'Pillow',
- ],
- 'act': LEROBOT_DEPENDENCIES + [
- 'Pillow; python_version>="3.10"',
- ],
- 'daft': [
- 'daft>=0.7.6; python_version>="3.10"',
- ],
- 'oss': [
- 'oss2>=2.18,<3',
- 'ossfs>=2021.8; python_version<"3.8"',
- 'ossfs>=2023; python_version>="3.8"'
- ],
- 'jindo': [
- 'oss2>=2.18,<3',
- 'pyjindosdk>=6.10.4',
- ],
- 'lance': [
- 'pylance>=0.20,<1; python_version>="3.9"',
- 'pylance>=0.10,<1; python_version>="3.8" and python_version<"3.9"'
- ],
- 'vortex': [
- 'vortex-data==0.70.0; python_version>="3.11"',
- ],
- 'mosaic': [
- 'paimon-mosaic>=0.1.0; python_version>="3.9"',
- ],
- 'lumina': [
- 'lumina-data>=0.1.0'
- ],
- 'vindex': [
- 'paimon-vindex==0.5.0; python_version>="3.9"',
- ],
- 'full-text': [
- 'paimon-ftindex==0.1.0; python_version>="3.8"',
- ],
- 'theta-sketch': [
- 'datasketches>=4,<5; python_version<"3.9"',
- 'datasketches>=5,<6; python_version>="3.9"',
- ],
- 'hll-sketch': [
- 'datasketches>=4,<5; python_version<"3.9"',
- 'datasketches>=5,<6; python_version>="3.9"',
- ],
- 'datafusion': [
- 'datafusion>=54,<55; python_version>="3.10"',
- ],
- 'sql': [
- 'pypaimon-rust>=0.3.0; python_version>="3.10"',
- 'datafusion>=54,<55; python_version>="3.10"',
- ],
- 'hdfs': [
- 'hdfs-native>=0.13,<1; python_version >= "3.10" and
platform_system != "Windows"',
- ],
- },
- description="Apache Paimon Python API",
- long_description=long_description,
- long_description_content_type="text/markdown",
- author="Apache Software Foundation",
- author_email="[email protected]",
- url="https://paimon.apache.org";,
- classifiers=[
- "Development Status :: 5 - Production/Stable",
- "License :: OSI Approved :: Apache Software License",
- "Programming Language :: Python :: 3.6",
- "Programming Language :: Python :: 3.7",
- "Programming Language :: Python :: 3.8",
- "Programming Language :: Python :: 3.9",
- "Programming Language :: Python :: 3.10",
- "Programming Language :: Python :: 3.11",
- "Programming Language :: Python :: 3.12",
- "Programming Language :: Python :: 3.13",
- ],
- python_requires=">=3.6",
+ **(_legacy_metadata() if sys.version_info < (3, 7) else {})
)
