This is an automated email from the ASF dual-hosted git repository. areusch pushed a commit to branch areusch/freeze-dependencies in repository https://gitbox.apache.org/repos/asf/tvm.git
commit a99fe89770a80b2bf5c02ac65a37327b13de789d Author: Andrew Reusch <[email protected]> AuthorDate: Mon May 23 18:33:03 2022 -0700 Fix all scripts which reference python3 in the docker containers. --- tests/lint/blocklint.sh | 6 ++++-- tests/lint/check_asf_header.sh | 4 ++-- tests/lint/cpplint.sh | 4 ++-- tests/lint/filter_untracked.py | 21 ++++++--------------- tests/lint/flake8.sh | 2 +- tests/lint/git-black.sh | 8 ++++---- tests/lint/jnilint.sh | 2 +- tests/lint/pylint.sh | 9 ++++----- tests/scripts/setup-pytest-env.sh | 4 ++-- tests/scripts/task_lint.sh | 5 ++++- tests/scripts/task_mypy.sh | 20 ++++++++++---------- 11 files changed, 40 insertions(+), 45 deletions(-) diff --git a/tests/lint/blocklint.sh b/tests/lint/blocklint.sh index 7525bfa64c..9222ef82d1 100755 --- a/tests/lint/blocklint.sh +++ b/tests/lint/blocklint.sh @@ -25,8 +25,10 @@ do if ! [ "$dir" == "3rdparty" ]; then for subdir in $(find $dir -type d -print) do - blocklint --blocklist blacklist,whitelist,white\ box,master\ ,\ master,master_,_master,slave $subdir \ - --skip-files tests/lint/blocklint.sh,tests/lint/pylintrc,conda/recipe/meta.yaml,rust/tvm-sys/build.rs,docs/topic/vta/dev/hardware.rst,src/target/source/codegen_vhls.cc,tests/micro/zephyr/test_utils.py + "${TVM_VENV}/bin/blocklint" \ + --blocklist blacklist,whitelist,white\ box,master\ ,\ master,master_,_master,slave \ + $subdir \ + --skip-files tests/lint/blocklint.sh,tests/lint/pylintrc,conda/recipe/meta.yaml,rust/tvm-sys/build.rs,docs/topic/vta/dev/hardware.rst,src/target/source/codegen_vhls.cc,tests/micro/zephyr/test_utils.py done fi done diff --git a/tests/lint/check_asf_header.sh b/tests/lint/check_asf_header.sh index 579069bb5b..bf71453899 100755 --- a/tests/lint/check_asf_header.sh +++ b/tests/lint/check_asf_header.sh @@ -52,7 +52,7 @@ if [ ${filter_untracked} -eq 1 ]; then echo "NOTE: --local flag present, filtering untracked files" processed_rat_output="${rat_output}-processed" cat ${rat_output} | sed 's/^== File: //g' | \ - python3 $(dirname "$0")/filter_untracked.py | \ + "${TVM_VENV}/bin/python3" $(dirname "$0")/filter_untracked.py | \ sed 's/^/== File: /g' >"${processed_rat_output}" rat_output="${processed_rat_output}" fi @@ -63,7 +63,7 @@ if [ $success -eq 0 ]; then echo "No files violate ASF header check" else if [ $fix -eq 1 ]; then - python3 tests/lint/add_asf_header.py "${rat_output}" + "${TVM_VENV}/bin/python3" tests/lint/add_asf_header.py "${rat_output}" echo "Ran add_asf_header.py to fix the following files:" else echo "Need to add ASF header to the following files." diff --git a/tests/lint/cpplint.sh b/tests/lint/cpplint.sh index 6c01f0eb0a..34908ff6df 100755 --- a/tests/lint/cpplint.sh +++ b/tests/lint/cpplint.sh @@ -19,8 +19,8 @@ set -e echo "Running 2 cpplints (VTA and TVM)..." -python3 3rdparty/dmlc-core/scripts/lint.py --quiet vta cpp vta/include vta/src -python3 3rdparty/dmlc-core/scripts/lint.py --quiet tvm cpp \ +"${TVM_VENV}/bin/python3" 3rdparty/dmlc-core/scripts/lint.py --quiet vta cpp vta/include vta/src +"${TVM_VENV}/bin/python3" 3rdparty/dmlc-core/scripts/lint.py --quiet tvm cpp \ include src \ examples/extension/src examples/graph_executor/src \ tests/cpp tests/crt \ diff --git a/tests/lint/filter_untracked.py b/tests/lint/filter_untracked.py index 3227bc3f18..990b82b8ae 100644 --- a/tests/lint/filter_untracked.py +++ b/tests/lint/filter_untracked.py @@ -22,27 +22,18 @@ import subprocess import sys -def check_output(args, **kw): - proc = subprocess.Popen(args, **kw, stdout=subprocess.PIPE) - out, _ = proc.communicate() - if proc.returncode: - sys.stderr.write("exited with code %d: %s\n" % (proc.returncode, " ".join(args))) - sys.exit(2) - - if sys.version_info[0] == 2: - return unicode(out, "utf-8") - else: - return str(out, "utf-8") - - def main(): script_dir = os.path.dirname(__file__) or os.getcwd() - toplevel_dir = check_output(["git", "rev-parse", "--show-toplevel"], cwd=script_dir).strip("\n") + toplevel_dir = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], cwd=script_dir, encoding="utf-8" + ).strip("\n") # NOTE: --ignore-submodules because this can drag in some problems related to mounting a git # worktree in the docker VM in a different location than it exists on the host. The problem # isn't quite clear, but anyhow it shouldn't be necessary to filter untracked files in # submodules here. - git_status_output = check_output(["git", "status", "-s", "--ignored"], cwd=toplevel_dir) + git_status_output = subprocess.check_output( + ["git", "status", "-s", "--ignored"], encoding="utf-8", cwd=toplevel_dir + ) untracked = [ line[3:] for line in git_status_output.split("\n") diff --git a/tests/lint/flake8.sh b/tests/lint/flake8.sh index 63dad38166..36b891fc78 100755 --- a/tests/lint/flake8.sh +++ b/tests/lint/flake8.sh @@ -18,4 +18,4 @@ set -e -python3 -m flake8 . --count --select=E9,F63,F7 --show-source --statistics +"${TVM_VENV}/bin/python3" -m flake8 . --count --select=E9,F63,F7 --show-source --statistics diff --git a/tests/lint/git-black.sh b/tests/lint/git-black.sh index 647aba9540..af7aa099bc 100755 --- a/tests/lint/git-black.sh +++ b/tests/lint/git-black.sh @@ -49,13 +49,13 @@ done export LC_ALL=C.UTF-8 export LANG=C.UTF-8 -if [ ! -x "$(command -v black)" ]; then +if [ ! -x "$(command -v "${TVM_VENV}/bin/black")" ]; then echo "Cannot find black" exit 1 fi # Print out specific version -VERSION=$(black --version) +VERSION=$("${TVM_VENV}/bin/python3" -m black --version) echo "black version: $VERSION" # Compute Python files which changed to compare. @@ -74,8 +74,8 @@ fi if [[ "$INPLACE_FORMAT" == "true" ]]; then echo "Running black on Python files against revision" $REVISION: - python3 -m black ${FILES[@]} + "${TVM_VENV}/bin/python3" -m black ${FILES[@]} else echo "Running black in checking mode" - python3 -m black --diff --check ${FILES[@]} + "${TVM_VENV}/bin/python3" -m black --diff --check ${FILES[@]} fi diff --git a/tests/lint/jnilint.sh b/tests/lint/jnilint.sh index ec359a5b89..2768bed875 100755 --- a/tests/lint/jnilint.sh +++ b/tests/lint/jnilint.sh @@ -18,4 +18,4 @@ set -e -python3 3rdparty/dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src +"${TVM_VENV}/bin/python3" 3rdparty/dmlc-core/scripts/lint.py tvm4j-jni cpp jvm/native/src diff --git a/tests/lint/pylint.sh b/tests/lint/pylint.sh index b442c33c0f..0e04e94687 100755 --- a/tests/lint/pylint.sh +++ b/tests/lint/pylint.sh @@ -17,8 +17,7 @@ # under the License. set -euxo pipefail -python3 -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc -python3 -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc -python3 -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc -python3 -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc - +"${TVM_VENV}/bin/python3" -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc +"${TVM_VENV}/bin/python3" -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc +"${TVM_VENV}/bin/python3" -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc +"${TVM_VENV}/bin/python3" -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index 63145c9909..d0a2ab17bb 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -39,7 +39,7 @@ function cleanup() { set +x if [ "${#pytest_errors[@]}" -gt 0 ]; then echo "These pytest invocations failed, the results can be found in the Jenkins 'Tests' tab or by scrolling up through the raw logs here." - python3 tests/scripts/pytest_wrapper.py "${pytest_errors[@]}" + "${TVM_VENV}/bin/python3" tests/scripts/pytest_wrapper.py "${pytest_errors[@]}" exit 1 fi set -x @@ -59,7 +59,7 @@ function run_pytest() { suite_name="${test_suite_name}-${ffi_type}" exit_code=0 - TVM_FFI=${ffi_type} python3 -m pytest \ + TVM_FFI=${ffi_type} "${TVM_VENV}/bin/python3" -m pytest \ -o "junit_suite_name=${suite_name}" \ "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \ "--junit-prefix=${ffi_type}" \ diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index 8fbba52662..5cc48553b0 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -28,6 +28,8 @@ trap cleanup 0 # These shards are solely for CI to enable the lint job to have some parallelism. function shard1 { + . "${TVM_VENV}/bin/activate" + echo "Convert scripts to Python..." tests/scripts/task_convert_scripts_to_python.sh @@ -57,6 +59,8 @@ function shard1 { } function shard2 { + . "${TVM_VENV}/bin/activate" + echo "Linting the Python code with pylint..." tests/lint/pylint.sh @@ -87,4 +91,3 @@ else shard1 shard2 fi - diff --git a/tests/scripts/task_mypy.sh b/tests/scripts/task_mypy.sh index 1ef7db5894..2cbe0ed723 100755 --- a/tests/scripts/task_mypy.sh +++ b/tests/scripts/task_mypy.sh @@ -21,26 +21,26 @@ set -euxo pipefail source tests/scripts/setup-pytest-env.sh echo "Checking MyPy Type defs in the TensorIR schedule package." -mypy --check-untyped-defs python/tvm/tir/schedule +"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/tir/schedule echo "Checking MyPy Type defs in the meta schedule package." -mypy --check-untyped-defs python/tvm/meta_schedule +"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/meta_schedule echo "Checking MyPy Type defs in the analysis package." -mypy --check-untyped-defs python/tvm/tir/analysis/ +"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/tir/analysis/ echo "Checking MyPy Type defs in the transform package." -mypy --check-untyped-defs python/tvm/tir/transform/ +"${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/tir/transform/ echo "Checking MyPy Type defs in the TIR package with unittest" -MYPYPATH=$TVM_PATH/python mypy --check-untyped-defs tests/python/unittest/test_tvmscript_type.py +MYPYPATH=$TVM_PATH/python "${TVM_VENV}/bin/mypy" --check-untyped-defs tests/python/unittest/test_tvmscript_type.py echo "Checking MyPy Type defs in tvm.relay.op.contrib" -mypy --disallow-untyped-defs python/tvm/relay/op/contrib/cublas.py -mypy --disallow-untyped-defs python/tvm/relay/op/contrib/cudnn.py -mypy --disallow-untyped-defs python/tvm/relay/op/contrib/te_target.py -mypy --disallow-untyped-defs python/tvm/relay/op/contrib/tensorrt.py +"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/cublas.py +"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/cudnn.py +"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/te_target.py +"${TVM_VENV}/bin/mypy" --disallow-untyped-defs python/tvm/relay/op/contrib/tensorrt.py #TODO(@mikepapadim): This is failing atm # echo "Checking MyPy Type defs in the tvm.relay.backend.contrib.ethosu package." -# mypy --check-untyped-defs python/tvm/relay/backend/contrib/ethosu/ +# "${TVM_VENV}/bin/mypy" --check-untyped-defs python/tvm/relay/backend/contrib/ethosu/
