This is an automated email from the ASF dual-hosted git repository.
bohan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 2cbd35d5d1 [PYTHON] Build cython by default (#17637)
2cbd35d5d1 is described below
commit 2cbd35d5d1d0ff18459af2584241db50fe5e84e7
Author: Tianqi Chen <[email protected]>
AuthorDate: Wed Feb 12 12:43:09 2025 -0500
[PYTHON] Build cython by default (#17637)
Historically we have been maintaining cython and ctypes for FFI.
One reason was cython was not that available then.
Cython have quite good availability now and also enjoys great
perf benefit. This PR changes the CMake to make cython as required
dependency of the python package and build cython by default.
It also removes ctypes testcases in favor of all cython tests.
If things go well, we can consider remove ctypes and focus on
cython support onward.
---
CMakeLists.txt | 12 ++++++++++
Makefile | 1 -
apps/benchmark/adreno/bench.sh | 2 --
python/setup.py | 29 ++++++++----------------
tests/scripts/setup-pytest-env.sh | 9 ++++----
tests/scripts/task_python_adreno.sh | 8 +++----
tests/scripts/task_python_arm_compute_library.sh | 9 ++++----
tests/scripts/task_python_docs.sh | 9 ++++++--
tests/scripts/task_python_hexagon.sh | 9 +++++---
tests/scripts/task_python_integration.sh | 17 ++++++--------
tests/scripts/task_python_nightly.sh | 6 ++---
tests/scripts/task_python_topi.sh | 6 ++---
tests/scripts/task_python_unittest.sh | 13 +++++------
tests/scripts/task_python_unittest_gpuonly.sh | 2 +-
tests/scripts/unity/task_python_relax.sh | 3 ++-
15 files changed, 68 insertions(+), 67 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 757b0d1a89..f43eab264d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -819,6 +819,18 @@ endif()
# Custom targets
add_custom_target(runtime DEPENDS tvm_runtime)
+# By default add cython to all build
+find_package(Python)
+if(NOT DEFINED ENV{CONDA_BUILD})
+ message(STATUS ${CMAKE_CURRENT_BINARY_DIR})
+ add_custom_target(
+ tvm_cython ALL
+ ${Python_EXECUTABLE} setup.py build_ext --inplace
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/python
+ )
+ add_dependencies(tvm_cython tvm)
+endif()
+
# Installation rules
install(TARGETS tvm EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
install(TARGETS tvm_runtime EXPORT ${PROJECT_NAME}Targets DESTINATION
lib${LIB_SUFFIX})
diff --git a/Makefile b/Makefile
index 3a880b94a5..5134c949ea 100644
--- a/Makefile
+++ b/Makefile
@@ -21,7 +21,6 @@
lint pylint cpplint \
cppdoc docs \
web webclean \
- cython cython3 cyclean \
clean
.SECONDEXPANSION:
diff --git a/apps/benchmark/adreno/bench.sh b/apps/benchmark/adreno/bench.sh
index 7f9adeea52..2c8ff28820 100755
--- a/apps/benchmark/adreno/bench.sh
+++ b/apps/benchmark/adreno/bench.sh
@@ -48,8 +48,6 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; }" 0
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
-# Test TVM
-make cython3
if [ "texture" == $1 ] ; then
python3 apps/benchmark/adreno/adreno_gpu_bench_texture.py --host
${TVM_TRACKER_HOST} --port ${TVM_TRACKER_PORT} --rpc-key ${RPC_DEVICE_KEY}
diff --git a/python/setup.py b/python/setup.py
index 55f29c651a..6cca9060bd 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -20,7 +20,6 @@ import os
import pathlib
import shutil
import sys
-import sysconfig
from setuptools import find_packages
from setuptools.dist import Distribution
@@ -133,24 +132,17 @@ __version__ = git_describe_version(__version__)
def config_cython():
"""Try to configure cython and return cython configuration"""
- if FFI_MODE not in ("cython"):
- if os.name == "nt" and not CONDA_BUILD:
- print("WARNING: Cython is not supported on Windows, will compile
without cython module")
- return []
- sys_cflags = sysconfig.get_config_var("CFLAGS")
- if sys_cflags and "i386" in sys_cflags and "x86_64" in sys_cflags:
- print("WARNING: Cython library may not be compiled correctly with
both i386 and x64")
- return []
+ # Enforce cython unless FFI_MODE is explicitly set to ctypes
+ # we might consider fully converge to cython later
+ if FFI_MODE == "ctypes":
+ return []
try:
from Cython.Build import cythonize
- # from setuptools.extension import Extension
- if sys.version_info >= (3, 0):
- subdir = "_cy3"
- else:
- subdir = "_cy2"
+ subdir = "_cy3"
+
ret = []
- path = "tvm/_ffi/_cython"
+ cython_source = "tvm/_ffi/_cython"
extra_compile_args = ["-std=c++17",
"-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>"]
if os.name == "nt":
library_dirs = ["tvm", "../build/Release", "../build"]
@@ -166,7 +158,7 @@ def config_cython():
library_dirs = None
libraries = None
- for fn in os.listdir(path):
+ for fn in os.listdir(cython_source):
if not fn.endswith(".pyx"):
continue
ret.append(
@@ -186,10 +178,7 @@ def config_cython():
)
return cythonize(ret, compiler_directives={"language_level": 3})
except ImportError as error:
- if FFI_MODE == "cython":
- raise error
- print("WARNING: Cython is not installed, will compile without cython
module")
- return []
+ raise RuntimeError("Cython is not installed, please pip install
cython")
class BinaryDistribution(Distribution):
diff --git a/tests/scripts/setup-pytest-env.sh
b/tests/scripts/setup-pytest-env.sh
index 7bdfbdd1df..171ddbc2d0 100755
--- a/tests/scripts/setup-pytest-env.sh
+++ b/tests/scripts/setup-pytest-env.sh
@@ -47,15 +47,14 @@ function cleanup() {
trap cleanup 0
function run_pytest() {
+ ffi_type="cython"
set -e
- local ffi_type="$1"
- shift
local test_suite_name="$1"
shift
extra_args=( "$@" )
if [ -z "${ffi_type}" -o -z "${test_suite_name}" ]; then
- echo "error: run_pytest called incorrectly: run_pytest ${ffi_type}
${test_suite_name}" "${extra_args[@]}"
- echo "usage: run_pytest <FFI_TYPE> <TEST_SUITE_NAME> [pytest args...]"
+ echo "error: run_pytest called incorrectly: run_pytest
${test_suite_name}" "${extra_args[@]}"
+ echo "usage: run_pytest <TEST_SUITE_NAME> [pytest args...]"
exit 2
fi
@@ -85,7 +84,7 @@ function run_pytest() {
exit_code=0
set +e
- TVM_FFI=${ffi_type} python3 -m pytest \
+ python3 -m pytest \
-o "junit_suite_name=${suite_name}" \
"--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \
"--junit-prefix=${ffi_type}" \
diff --git a/tests/scripts/task_python_adreno.sh
b/tests/scripts/task_python_adreno.sh
index e20d1bea31..a435d588ae 100755
--- a/tests/scripts/task_python_adreno.sh
+++ b/tests/scripts/task_python_adreno.sh
@@ -57,8 +57,8 @@ trap "{ kill ${TRACKER_PID}; kill ${DEVICE_PID}; cleanup; }" 0
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
-# Test TVM
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
# The RPC to remote Android device has issue of hang after few tests with in
CI environments.
# Lets run them individually on fresh rpc session.
@@ -68,7 +68,7 @@ i=0
IFS=$'\n'
for node_id in $TEXTURE_TESTS; do
echo "$node_id"
- run_pytest ctypes "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i"
"$node_id" --reruns=0
+ run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-opencl-texture-$i" "$node_id"
--reruns=0
i=$((i+1))
done
@@ -77,7 +77,7 @@ CLML_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder
tests/python/contrib/te
i=0
for node_id in $CLML_TESTS; do
echo "$node_id"
- CXX=${TVM_NDK_CC} run_pytest ctypes
"$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i" "$node_id" --reruns=0
+ CXX=${TVM_NDK_CC} run_pytest "$TVM_INTEGRATION_TESTSUITE_NAME-openclml-$i"
"$node_id" --reruns=0
i=$((i+1))
done
diff --git a/tests/scripts/task_python_arm_compute_library.sh
b/tests/scripts/task_python_arm_compute_library.sh
index 51b453e2b8..520abb58cd 100755
--- a/tests/scripts/task_python_arm_compute_library.sh
+++ b/tests/scripts/task_python_arm_compute_library.sh
@@ -21,10 +21,9 @@ set -euxo pipefail
source tests/scripts/setup-pytest-env.sh
-# Rebuild cython
-# TODO(u99127): Enable cython tests.
-
find . -type f -path "*.pyc" | xargs rm -f
-make cython3
-run_pytest ctypes python-arm_compute_lib
tests/python/contrib/test_arm_compute_lib
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
+
+run_pytest python-arm_compute_lib tests/python/contrib/test_arm_compute_lib
diff --git a/tests/scripts/task_python_docs.sh
b/tests/scripts/task_python_docs.sh
index a72078d740..b217f692b4 100755
--- a/tests/scripts/task_python_docs.sh
+++ b/tests/scripts/task_python_docs.sh
@@ -46,7 +46,9 @@ clean_files() {
sphinx_precheck() {
clean_files
echo "PreCheck sphinx doc generation WARNINGS.."
- make cython3
+
+ # setup cython
+ cd python; python3 setup.py build_ext --inplace; cd ..
pushd docs
make clean
@@ -123,7 +125,10 @@ clean_files
# cleanup stale log files
find . -type f -path "*.log" | xargs rm -f
find . -type f -path "*.pyc" | xargs rm -f
-make cython3
+
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
+
cd docs
PYTHONPATH=$(pwd)/../python make htmldepoly SPHINXOPTS='-j auto' |& tee
/tmp/$$.log.txt
diff --git a/tests/scripts/task_python_hexagon.sh
b/tests/scripts/task_python_hexagon.sh
index ba125b161a..11e4e18274 100755
--- a/tests/scripts/task_python_hexagon.sh
+++ b/tests/scripts/task_python_hexagon.sh
@@ -26,7 +26,10 @@ if [ $# -ge 1 ] && [[ "$1" = "--device" ]]; then
fi
source tests/scripts/setup-pytest-env.sh
-make cython3
+
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
+
if [[ "${device_serial}" == "simulator" ]]; then
export TVM_TRACKER_PORT=9190
@@ -49,9 +52,9 @@ fi
export ANDROID_SERIAL_NUMBER=${device_serial}
if [ "${device_serial}" == "simulator" ]; then
- run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon
+ run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon
else
- run_pytest ctypes python-contrib-hexagon tests/python/contrib/test_hexagon
-n=$num_of_devices
+ run_pytest python-contrib-hexagon tests/python/contrib/test_hexagon
-n=$num_of_devices
fi
if [[ "${device_serial}" == "simulator" ]]; then
diff --git a/tests/scripts/task_python_integration.sh
b/tests/scripts/task_python_integration.sh
index da0d050083..51ef86d05e 100755
--- a/tests/scripts/task_python_integration.sh
+++ b/tests/scripts/task_python_integration.sh
@@ -33,23 +33,20 @@ fi
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
-# Test TVM
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-integration
tests/python/integration
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-integration
tests/python/integration
# forked is needed because the global registry gets contaminated
TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" \
- run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-relay
tests/python/relay --ignore=tests/python/relay/aot
+ run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay
--ignore=tests/python/relay/aot
# OpenCL texture test. Deselected specific tests that fails in CI
TVM_TEST_TARGETS="${TVM_RELAY_OPENCL_TEXTURE_TARGETS:-opencl}" \
- run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture
tests/python/relay/opencl_texture
+ run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture
tests/python/relay/opencl_texture
# Command line driver test
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
# Target test
-run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target
-
-# Do not enable OpenGL
-# run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-webgl tests/webgl
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-target tests/python/target
diff --git a/tests/scripts/task_python_nightly.sh
b/tests/scripts/task_python_nightly.sh
index f8423602bd..42cf343e71 100755
--- a/tests/scripts/task_python_nightly.sh
+++ b/tests/scripts/task_python_nightly.sh
@@ -20,10 +20,10 @@ set -euxo pipefail
source tests/scripts/setup-pytest-env.sh
-# Rebuild cython
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
-run_pytest cython python-topi-nightly tests/python/topi/nightly
+run_pytest python-topi-nightly tests/python/topi/nightly
diff --git a/tests/scripts/task_python_topi.sh
b/tests/scripts/task_python_topi.sh
index 3916e0a754..97030d6948 100755
--- a/tests/scripts/task_python_topi.sh
+++ b/tests/scripts/task_python_topi.sh
@@ -26,10 +26,10 @@ export TVM_TEST_TARGETS="llvm; cuda"
export TVM_BIND_THREADS=0
export OMP_NUM_THREADS=1
-# Rebuild cython
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
-run_pytest cython python-topi tests/python/topi/
+run_pytest python-topi tests/python/topi/
diff --git a/tests/scripts/task_python_unittest.sh
b/tests/scripts/task_python_unittest.sh
index 4a13c6ce1e..b84694cfd5 100755
--- a/tests/scripts/task_python_unittest.sh
+++ b/tests/scripts/task_python_unittest.sh
@@ -22,7 +22,9 @@ source tests/scripts/setup-pytest-env.sh
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
-make cython3
+
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
# NOTE: also set by task_python_unittest_gpuonly.sh.
if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
@@ -30,8 +32,7 @@ if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
fi
# First run minimal test on both ctypes and cython.
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-0
tests/python/all-platform-minimal-test
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-1
tests/python/all-platform-minimal-test
+run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test
tests/python/all-platform-minimal-test
# Then run all unittests on both ctypes and cython.
TEST_FILES=(
@@ -52,11 +53,9 @@ TEST_FILES=(
"tir-usmp"
"tvmscript"
"usmp"
+ "ci"
)
for TEST_FILE in ${TEST_FILES[@]}; do
- run_pytest cython ${TEST_FILE}-1, tests/python/${TEST_FILE}
+ run_pytest ${TEST_FILE}, tests/python/${TEST_FILE}
done
-
-# Then run CI tests
-run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-ci tests/python/ci
diff --git a/tests/scripts/task_python_unittest_gpuonly.sh
b/tests/scripts/task_python_unittest_gpuonly.sh
index e68fcba25c..6c4e642b0c 100755
--- a/tests/scripts/task_python_unittest_gpuonly.sh
+++ b/tests/scripts/task_python_unittest_gpuonly.sh
@@ -33,4 +33,4 @@ export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan
source tests/scripts/setup-pytest-env.sh
-run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-1
tests/python/codegen/test_target_codegen_vulkan.py
+run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-1
tests/python/codegen/test_target_codegen_vulkan.py
diff --git a/tests/scripts/unity/task_python_relax.sh
b/tests/scripts/unity/task_python_relax.sh
index 86d917c655..28dd78bf6b 100755
--- a/tests/scripts/unity/task_python_relax.sh
+++ b/tests/scripts/unity/task_python_relax.sh
@@ -25,7 +25,8 @@ export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}"
export TVM_BIND_THREADS=0
export TVM_NUM_THREADS=2
-make cython3
+# setup cython
+cd python; python3 setup.py build_ext --inplace; cd ..
# Run Relax tests
TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm}" pytest tests/python/relax
