This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 0ed61f6 refactor: move `Makefile` to root directory (#283)
0ed61f6 is described below
commit 0ed61f641f67103164c9503ddcaaa03011a0db9a
Author: verma nakul <[email protected]>
AuthorDate: Sun Feb 2 08:29:08 2025 +0530
refactor: move `Makefile` to root directory (#283)
---------
Co-authored-by: Shiyan Xu <[email protected]>
---
.github/workflows/ci.yml | 11 +++--------
.github/workflows/code.yml | 6 ++----
CONTRIBUTING.md | 26 ++++++++------------------
python/Makefile => Makefile | 23 +++++++++++++----------
demo/run_demo.sh | 2 +-
python/tests/conftest.py | 6 +++---
6 files changed, 30 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ebdfde0..4bfe7f2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -64,7 +64,6 @@ jobs:
- name: Cache dependencies
uses: Swatinem/rust-cache@v2
- name: Rust unit tests with coverage report
- #
https://github.com/xd009642/tarpaulin/issues/1092#issuecomment-1407739176
run: cargo tarpaulin --engine llvm --no-dead-code --no-fail-fast
--all-features --workspace -o xml --output-dir ./cov-reports --skip-clean
- name: Upload coverage report
uses: actions/upload-artifact@v4
@@ -91,21 +90,17 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
cache: pip
- cache-dependency-path: pyproject.toml
+ cache-dependency-path: python/pyproject.toml
- name: Setup Python venv
- working-directory: ./python
run: |
make setup-venv
source venv/bin/activate
make develop
- name: Python unit tests with coverage report
run: |
- pushd python
source venv/bin/activate
- coverage run --include 'hudi/*' -m pytest -v
- # move to parent so the reported file paths will match the actual
repo paths
- popd
- coverage xml --data-file=python/.coverage -o
./cov-reports/cov-report-python-tests-${{ join(matrix.*, '-') }}.xml
+ coverage run --include 'python/hudi/*' -m pytest -v
+ coverage xml --data-file=.coverage -o
./cov-reports/cov-report-python-tests-${{ join(matrix.*, '-') }}.xml
- name: Upload coverage report
uses: actions/upload-artifact@v4
with:
diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml
index a8bb1d6..b34ce4f 100644
--- a/.github/workflows/code.yml
+++ b/.github/workflows/code.yml
@@ -36,24 +36,22 @@ jobs:
uses: apache/skywalking-eyes/[email protected]
- name: Check rust code style
- run: cd python && make check-rust
+ run: make check-rust
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: '3.9'
cache: pip
- cache-dependency-path: pyproject.toml
+ cache-dependency-path: python/pyproject.toml
- name: Install python linter dependencies
- working-directory: ./python
run: |
make setup-venv
source venv/bin/activate
pip install ruff==0.5.2 mypy==1.10.1
- name: Check python code style
- working-directory: ./python
run: |
source venv/bin/activate
make check-python
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 696872d..f79d66d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -49,8 +49,7 @@ view, instead, they will be linked to the corresponding
issues.
## Commonly used dev commands
-For most of the time, use dev commands specified in
[`python/Makefile`](./python/Makefile), it applies to both Python
-and Rust modules. You don't need to `cd` to the root directory and run `cargo`
commands.
+For most of the time, use dev commands specified in the [`Makefile`](Makefile).
To setup python virtual env, run
@@ -59,13 +58,10 @@ make setup-venv
```
> [!NOTE]
-> This will run `python` command to setup the virtual environment. You can
either change that to `python3.X`,
-> or simply alias `python` to your local `python3.X` installation, for example:
-> ```shell
-> echo "alias
python=/Library/Frameworks/Python.framework/Versions/3.12/bin/python3" >>
~/.zshrc`
-> ```
+> This will run `python3` command to set up the virtual environment in `venv/`.
+> Activate the virtual environment by running `source venv/bin/activate` for
example.
-Once activate virtual env, build the project for development by
+Once a virtual environment is activated, build the project for development by
```shell
make develop
@@ -96,24 +92,18 @@ For Python,
# For all tests
make test-python
# or
-pytest -s
+pytest -s python/tests
# For a specific test case
-pytest tests/test_table_read.py -s -k "test_read_table_has_correct_schema"
+pytest python/tests/test_table_read.py -s -k
"test_read_table_has_correct_schema"
```
## Before creating a pull request
-Run test commands to make sure the code is working as expected:
+Run the below command and fix issues if any:
```shell
-make test
-```
-
-Run check commands and follow the suggestions to fix the code:
-
-```shell
-make check
+make format check test
```
## Create a pull request
diff --git a/python/Makefile b/Makefile
similarity index 80%
rename from python/Makefile
rename to Makefile
index 948631e..5f826b5 100644
--- a/python/Makefile
+++ b/Makefile
@@ -15,16 +15,19 @@
# specific language governing permissions and limitations
# under the License.
+SHELL := /bin/bash
+
.DEFAULT_GOAL := help
VENV := venv
-MATURIN_VERSION := $(shell grep 'requires =' pyproject.toml | cut -d= -f2- |
tr -d '[ "]')
+PYTHON_DIR = python
+MATURIN_VERSION := $(shell grep 'requires =' $(PYTHON_DIR)/pyproject.toml |
cut -d= -f2- | tr -d '[ "]')
PACKAGE_VERSION := $(shell grep version Cargo.toml | head -n 1 | awk '{print
$$3}' | tr -d '"' )
.PHONY: setup-venv
setup-venv: ## Setup the virtualenv
$(info --- Setup virtualenv ---)
- python -m venv $(VENV)
+ python3 -m venv $(VENV)
.PHONY: setup
setup: ## Setup the requirements
@@ -34,12 +37,12 @@ setup: ## Setup the requirements
.PHONY: build
build: setup ## Build Python binding of hudi-rs
$(info --- Build Python binding ---)
- maturin build $(MATURIN_EXTRA_ARGS)
+ maturin build $(MATURIN_EXTRA_ARGS) -m $(PYTHON_DIR)/Cargo.toml
.PHONY: develop
develop: setup ## Install Python binding of hudi-rs
$(info --- Develop with Python binding ---)
- maturin develop --extras=devel,pandas $(MATURIN_EXTRA_ARGS)
+ maturin develop --extras=devel,pandas $(MATURIN_EXTRA_ARGS) -m
$(PYTHON_DIR)/Cargo.toml
.PHONY: format
format: format-rust format-python ## Format Rust and Python code
@@ -52,7 +55,7 @@ format-rust: ## Format Rust code
.PHONY: format-python
format-python: ## Format Python code
$(info --- Format Python code ---)
- ruff format .
+ ruff format $(PYTHON_DIR)
.PHONY: check
check: check-rust check-python ## Run check on Rust and Python
@@ -60,18 +63,18 @@ check: check-rust check-python ## Run check on Rust and
Python
.PHONY: check-rust
check-rust: ## Run check on Rust
$(info --- Check Rust clippy ---)
- cargo clippy --all-targets --all-features --workspace -- -D warnings
+ cargo clippy --all-targets --all-features --workspace --no-deps -- -D
warnings
$(info --- Check Rust format ---)
cargo fmt --all -- --check
.PHONY: check-python
check-python: ## Run check on Python
$(info --- Check Python format ---)
- ruff format --check --diff .
+ ruff format --check --diff $(PYTHON_DIR)
$(info --- Check Python linting ---)
- ruff check .
+ ruff check $(PYTHON_DIR)
$(info --- Check Python typing ---)
- mypy .
+ pushd $(PYTHON_DIR); mypy .; popd
.PHONY: test
test: test-rust test-python ## Run tests on Rust and Python
@@ -84,4 +87,4 @@ test-rust: ## Run tests on Rust
.PHONY: test-python
test-python: ## Run tests on Python
$(info --- Run Python tests ---)
- pytest -s
+ pytest -s $(PYTHON_DIR)
diff --git a/demo/run_demo.sh b/demo/run_demo.sh
index cec677b..8a9b275 100755
--- a/demo/run_demo.sh
+++ b/demo/run_demo.sh
@@ -36,7 +36,7 @@ fi
# install dependencies and run the demo apps
docker compose exec -T runner /bin/bash -c "
- cd /opt/hudi-rs/python && \
+ cd /opt/hudi-rs && \
make setup develop && \
cd /opt/hudi-rs/demo/sql-datafusion && ./run.sh &&\
cd /opt/hudi-rs/demo/table-api-python && ./run.sh && \
diff --git a/python/tests/conftest.py b/python/tests/conftest.py
index b1fd566..4152273 100644
--- a/python/tests/conftest.py
+++ b/python/tests/conftest.py
@@ -16,8 +16,8 @@
# under the License.
import os
+import pathlib
import zipfile
-from pathlib import Path
import pytest
@@ -34,7 +34,7 @@ def _extract_testing_table(zip_file_path, target_path) -> str:
]
)
def get_sample_table(request, tmp_path) -> str:
- fixture_path = "tests/table"
+ fixture_path = pathlib.Path(__file__).parent.joinpath("table")
table_name = request.param
- zip_file_path = Path(fixture_path).joinpath(f"{table_name}.zip")
+ zip_file_path = pathlib.Path(fixture_path).joinpath(f"{table_name}.zip")
return _extract_testing_table(zip_file_path, tmp_path)
