This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 69dc0588f76245ce9503e15c72219a0a646156fd Author: Joe McDonnell <[email protected]> AuthorDate: Sat Feb 7 12:54:28 2026 -0800 IMPALA-14620 (part 1): Remove the Python 2 impala-python virtualenv Previous changes stopped building the Python 2 impala-python virtualenv, but the code is still around. Since Python 2 is dead, this removes that support and simplifies the code: 1. It removes the impala-python Python 2 virtualenv. This removes py2-requirements.txt and folds py3-requirements.txt back into requirements.txt. This also removes various helper scripts for the Python 2 virtualenv. 2. This removes pylint and the check-pylint-py3k.sh script. Pylint is currently only used for the py3k check. This keeps isort separately. 3. It drops the Python 2 virtualenv package and support. 4. This drops IMPALA_USE_PYTHON3_TESTS and adjusts run-tests.py to use impala-python3 directly. 5. This drops the code for producing the Python 2 virtualenv for the shell (and associated packaging code). This also removes the test code for handling the Python 2 shell. This does not drop Python 2 support from impala-shell, so it is still possible to pip install it for Python 2. 6. This drops some code for detecting whether Python supports SSL, which was used for old versions of Python 2. Testing: - Ran a core job Change-Id: If28a8a0466614fc1d022e27c742dc3880509947a Reviewed-on: http://gerrit.cloudera.org:8080/23967 Reviewed-by: Michael Smith <[email protected]> Tested-by: Joe McDonnell <[email protected]> --- CMakeLists.txt | 10 +- bin/bootstrap_toolchain.py | 4 +- bin/check-pylint-py3k.sh | 137 ------------------------ bin/check-python-syntax.sh | 29 ------ bin/cmake_aux/create_virtualenv.sh | 19 +--- bin/impala-config.sh | 8 +- bin/impala-env-versioned-python | 26 ----- bin/impala-futurize | 21 ---- bin/impala-pip | 21 ---- bin/impala-py.test | 6 +- bin/impala-pylint | 21 ---- bin/impala-python | 23 ---- bin/impala-python-common.sh | 32 ------ bin/impala-python3-common.sh | 2 +- bin/impala-shell.sh | 10 +- bin/impala-virtualenv | 21 ---- bin/init-impala-python.sh | 26 +---- infra/python/bootstrap_virtualenv.py | 162 ++++++++--------------------- infra/python/deps/pip_download.py | 1 - infra/python/deps/py2-requirements.txt | 33 ------ infra/python/deps/py3-requirements.txt | 51 --------- infra/python/deps/requirements.txt | 22 +++- shell/CMakeLists.txt | 17 --- shell/packaging/make_shell_tarball.sh | 12 +-- tests/common/network.py | 6 -- tests/custom_cluster/test_client_ssl.py | 36 +------ tests/custom_cluster/test_thrift_socket.py | 12 --- tests/run-tests.py | 2 +- tests/shell/test_shell_commandline.py | 3 - tests/shell/test_shell_interactive.py | 2 - tests/shell/util.py | 8 -- 31 files changed, 84 insertions(+), 699 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9aaa132dc..75ec07d25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -536,19 +536,11 @@ add_custom_target(cscope ALL DEPENDS gen-deps COMMAND "${CMAKE_SOURCE_DIR}/bin/gen-cscope.sh" ) -# IMPALA-14606: Stop building impala_python (Python 2) by default. -# add_custom_target(impala_python ALL -# COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" -# ) - add_custom_target(impala_python3 ALL - COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" "-python3" + COMMAND "${CMAKE_SOURCE_DIR}/bin/init-impala-python.sh" ) set(IMPALA_PYTHON_INSTALLS "") -if (NOT $ENV{IMPALA_SYSTEM_PYTHON2} EQUAL "") - list(APPEND IMPALA_PYTHON_INSTALLS shell_python2_install) -endif() if (NOT $ENV{IMPALA_SYSTEM_PYTHON3} EQUAL "") list(APPEND IMPALA_PYTHON_INSTALLS shell_python3_install) endif() diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py index f3f0c8381..e1a074099 100755 --- a/bin/bootstrap_toolchain.py +++ b/bin/bootstrap_toolchain.py @@ -478,11 +478,9 @@ def get_toolchain_downloads(): "glog", "gperftools", "jwt-cpp", "libev", "libunwind", "lz4", "mold", "openldap", "opentelemetry-cpp", "orc", "protobuf", "python", "rapidjson", "re2", "snappy", "tpc-h", "tpc-ds", "zlib", "zstd"]] - python3_package = ToolchainPackage( - "python", explicit_version=os.environ.get("IMPALA_PYTHON3_VERSION")) gtest_package = ToolchainPackage( "googletest", explicit_version=os.environ.get("IMPALA_GTEST_VERSION")) - toolchain_packages += [python3_package, gtest_package] + toolchain_packages += [gtest_package] toolchain_packages += get_unique_toolchain_downloads( ["thrift:cpp", "thrift:java", "thrift:py"]) protobuf_package_clang = ToolchainPackage( diff --git a/bin/check-pylint-py3k.sh b/bin/check-pylint-py3k.sh deleted file mode 100755 index 305ab990a..000000000 --- a/bin/check-pylint-py3k.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -BINDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -# To allow incrementally banning individual pylint checks, this uses grep -# expressions to match banned pylint warnings. The grep expressions are stored -# in the bin/banned_py3k_warnings.txt file. -BANNED_PY3K_WARNINGS="${BINDIR}/banned_py3k_warnings.txt" - -function print_usage { - echo "check-pylink-py3k.sh : Checks eligible python files for pylint py3k compliance." - echo "Fails if the python files have py3k warnings that match the patterns in " - echo "bin/banned_py3k_warnings.txt." - echo "[--error_output_file] : (optional) Also output the errors to a file" - echo "[--warning_output_file] : (optional) Also output the warnings to a file" -} - -ERROR_OUTPUT_FILE="" -WARNING_OUTPUT_FILE="" -while [ -n "$*" ] -do - case "$1" in - --error_output_file) - ERROR_OUTPUT_FILE="${2-}" - shift; - ;; - --warning_output_file) - WARNING_OUTPUT_FILE="${2-}" - shift; - ;; - --help|*) - print_usage - exit 1 - ;; - esac - shift -done - -pushd ${IMPALA_HOME} > /dev/null 2>&1 - -OUTPUT_TMP_DIR=$(mktemp -d) -PYLINT_OUTPUT_FILE="${OUTPUT_TMP_DIR}/pylint_output.txt" -ERROR_OUTPUT_TMP_FILE="${OUTPUT_TMP_DIR}/error_output_tmp.txt" -WARNING_OUTPUT_TMP_FILE="${OUTPUT_TMP_DIR}/warning_output_tmp.txt" - -RETCODE=0 -for file in $(git ls-files '**/*.py'); do - # Skip the shell entirely (but cover tests/shell) - if [[ "${file}" =~ "shell/" && ! "${file}" =~ "tests/shell" ]]; then - continue - fi - # Ignore files that are created to run with python3. - FIRST_LINE=$(head -n1 ${file}) - if [[ "${file}: ${FIRST_LINE}" =~ "#!" ]]; then - if [[ "${FIRST_LINE}" =~ "python3" ]]; then - >&2 echo "SKIPPING: ${file} is already using python3: ${FIRST_LINE}" - continue - fi - if [[ "${FIRST_LINE}" =~ "/bin/bash" ]]; then - >&2 echo "SKIPPING: ${file} is a weird bash/python hybrid: ${FIRST_LINE}" - continue - fi - fi - - >&2 echo "PROCESSING: ${file}" - - # -s n (skip score for each file) - # --exit-zero: don't fail - impala-pylint -s n --exit-zero --py3k ${file} >> ${PYLINT_OUTPUT_FILE} -done - -touch "${ERROR_OUTPUT_TMP_FILE}" -touch "${WARNING_OUTPUT_TMP_FILE}" - -# Hitting a banned py3k warning will cause this to return an error -echo "" -echo "" -if grep -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" > /dev/null 2>&1 ; then - echo "ERROR: Some python files contain these banned pylint warnings:" | \ - tee "${ERROR_OUTPUT_TMP_FILE}" - grep -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" | \ - tee -a "${ERROR_OUTPUT_TMP_FILE}" - RETCODE=1 -else - echo "No errors found" | tee "${ERROR_OUTPUT_TMP_FILE}" -fi - -if [[ -n "${ERROR_OUTPUT_FILE}" ]]; then - cp "${ERROR_OUTPUT_TMP_FILE}" "${ERROR_OUTPUT_FILE}" -fi - -# The remaining py3k warnings are interesting, but they are not yet enforced. -# Pylint produces annoying lines like "************* Module X", so try to filter those out -echo "" -echo "" -if grep -v -e '\*\*\*\*' -f "${BANNED_PY3K_WARNINGS}" \ - "${PYLINT_OUTPUT_FILE}" > /dev/null 2>&1 ; then - echo "WARNING: Some python files contain these unenforced pylint warnings:" | \ - tee "${WARNING_OUTPUT_TMP_FILE}" - grep -v -e '\*\*\*\*' -f "${BANNED_PY3K_WARNINGS}" "${PYLINT_OUTPUT_FILE}" | \ - tee -a "${WARNING_OUTPUT_TMP_FILE}" - - echo "WARNING SUMMARY table:" - cat "${WARNING_OUTPUT_TMP_FILE}" | grep -v "WARNING" | cut -d: -f4- | \ - sed 's#^ ##' | sort | uniq -c -else - echo "No warnings found" | tee "${WARNING_OUTPUT_TMP_FILE}" -fi - -if [[ -n "${WARNING_OUTPUT_FILE}" ]]; then - cp "${WARNING_OUTPUT_TMP_FILE}" "${WARNING_OUTPUT_FILE}" -fi - -rm -rf "${OUTPUT_TMP_DIR}" - -popd > /dev/null 2>&1 - -exit ${RETCODE} diff --git a/bin/check-python-syntax.sh b/bin/check-python-syntax.sh index 5576297ff..7b38e05f2 100755 --- a/bin/check-python-syntax.sh +++ b/bin/check-python-syntax.sh @@ -23,35 +23,6 @@ pushd ${IMPALA_HOME} > /dev/null 2>&1 RETCODE=0 for file in $(git ls-files '**/*.py'); do - # Skip the shell's ext-py code - if [[ "${file}" =~ "shell/ext-py" ]]; then - continue - fi - # Skip the shell's pkg_resources.py - if [[ "${file}" == "shell/legacy/pkg_resources.py" ]]; then - continue - fi - - # Python 2 checks - # -l = no recursion - # -q = only print errors - # -f = force recompile - # Ignore files that are created to run with python3. - FIRST_LINE=$(head -n1 ${file}) - if [[ "${FIRST_LINE}" =~ "python3" ]]; then - >&2 echo "SKIPPING: ${file} is already using python3: ${FIRST_LINE}" - elif ! python2 -m compileall -l -q -f ${file} > /dev/null 2>&1; then - RETCODE=1 - echo "Python 2 compilation failed for ${file}:" - set +e - python2 -m compileall -l -q -f ${file} - set -e - fi - # Clean up the .pyc files generated by compilation - if [[ -f "${file}c" ]]; then - rm "${file}c" - fi - # Python 3 checks # -l = no recursion # -q = only print errors diff --git a/bin/cmake_aux/create_virtualenv.sh b/bin/cmake_aux/create_virtualenv.sh index 2afb4dc69..f47ee5c6c 100755 --- a/bin/cmake_aux/create_virtualenv.sh +++ b/bin/cmake_aux/create_virtualenv.sh @@ -34,19 +34,10 @@ VENV_DIR=$2 IS_PY36_OR_HIGHER=$(${PYTHON_EXE} -c "import sys; print(\ 'true' if sys.version_info.major >= 3 and sys.version_info.minor >= 6 else 'false')") -# If using Python >= 3.6, try to use the builtin venv package. -if $IS_PY36_OR_HIGHER ; then - if ${PYTHON_EXE} -m venv ${VENV_DIR} ; then - # Success - exit 0 - fi - # Failure - echo "WARNING: Tried to create virtualenv with Python3's venv module and failed." - echo "Falling back to old impala-virtualenv path..." - # Remove the directory that Python3 venv created, so impala-virtualenv can start - # from a clean slate. - rm -rf ${VENV_DIR} +if ! $IS_PY36_OR_HIGHER ; then + echo "ERROR: $0 only supports Python >= 3.6" + exit 1 fi -# Fall back to using the old impala-virtualenv method -impala-virtualenv --python ${PYTHON_EXE} ${VENV_DIR} +# Rely on the builtin venv package for Python >= 3.6 +${PYTHON_EXE} -m venv ${VENV_DIR} diff --git a/bin/impala-config.sh b/bin/impala-config.sh index ec495b517..0c9808f0c 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -189,9 +189,8 @@ export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=42.5.6 unset IMPALA_POSTGRES_JDBC_DRIVER_URL export IMPALA_MYSQL_JDBC_DRIVER_VERSION=8.2.0 unset IMPALA_MYSQL_JDBC_DRIVER_URL -export IMPALA_PYTHON_VERSION=2.7.16 +export IMPALA_PYTHON_VERSION=3.8.18 unset IMPALA_PYTHON_URL -export IMPALA_PYTHON3_VERSION=3.8.18 export IMPALA_RAPIDJSON_VERSION=1.1.0-p1 unset IMPALA_RAPIDJSON_URL export IMPALA_RE2_VERSION=2023-03-01 @@ -224,8 +223,6 @@ unset IMPALA_MOLD_URL export IMPALA_SIMBA_JDBC_DRIVER_VERSION=42-2.6.32.1041 # Find system python versions for testing -# IMPALA-14606: Stop building impala_python (Python 2) by default. -export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-}" export IMPALA_SYSTEM_PYTHON3="${IMPALA_SYSTEM_PYTHON3_OVERRIDE-$(command -v python3)}" # Additional Python versions to use when building the impala-shell prebuilt tarball @@ -350,8 +347,6 @@ export IMPALA_KERBERIZE=false unset IMPALA_TOOLCHAIN_KUDU_MAVEN_REPOSITORY unset IMPALA_TOOLCHAIN_KUDU_MAVEN_REPOSITORY_ENABLED -export IMPALA_USE_PYTHON3_TESTS=${IMPALA_USE_PYTHON3_TESTS:-true} - # Source the branch and local config override files here to override any # variables above or any variables below that allow overriding via environment # variable. @@ -1302,7 +1297,6 @@ echo "IMPALA_ICEBERG_VERSION = $IMPALA_ICEBERG_VERSION" echo "IMPALA_PAIMON_VERSION = $IMPALA_PAIMON_VERSION" echo "IMPALA_COS_VERSION = $IMPALA_COS_VERSION" echo "IMPALA_OBS_VERSION = $IMPALA_OBS_VERSION" -echo "IMPALA_SYSTEM_PYTHON2 = $IMPALA_SYSTEM_PYTHON2" echo "IMPALA_SYSTEM_PYTHON3 = $IMPALA_SYSTEM_PYTHON3" echo "IMPALA_BUILD_THREADS = $IMPALA_BUILD_THREADS" echo "IMPALA_LINK_THREADS = $IMPALA_LINK_THREADS" diff --git a/bin/impala-env-versioned-python b/bin/impala-env-versioned-python deleted file mode 100755 index 3bb8dbf42..000000000 --- a/bin/impala-env-versioned-python +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# -############################################################################## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -############################################################################## - -if [[ "${IMPALA_USE_PYTHON3_TESTS}" == "true" ]]; then - exec impala-python3 "$@" -else - exec impala-python "$@" -fi diff --git a/bin/impala-futurize b/bin/impala-futurize deleted file mode 100755 index b761b5212..000000000 --- a/bin/impala-futurize +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source "$(dirname "$0")/impala-python-common.sh" -exec "$PY_ENV_DIR/bin/futurize" "$@" diff --git a/bin/impala-pip b/bin/impala-pip deleted file mode 100755 index 6845a8f6b..000000000 --- a/bin/impala-pip +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source "$(dirname "$0")/impala-python-common.sh" -exec "$PY_ENV_DIR/bin/python" "$PY_ENV_DIR/bin/pip" "$@" diff --git a/bin/impala-py.test b/bin/impala-py.test index 6fd59d4f9..16dcc6fdc 100755 --- a/bin/impala-py.test +++ b/bin/impala-py.test @@ -19,9 +19,5 @@ # under the License. ############################################################################## -if [[ "${IMPALA_USE_PYTHON3_TESTS}" == "true" ]]; then - source $(dirname "$0")/impala-python3-common.sh -else - source $(dirname "$0")/impala-python-common.sh -fi +source $(dirname "$0")/impala-python3-common.sh exec "$PY_ENV_DIR/bin/py.test" "$@" diff --git a/bin/impala-pylint b/bin/impala-pylint deleted file mode 100755 index 012f08bc9..000000000 --- a/bin/impala-pylint +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source "$(dirname "$0")/impala-python3-common.sh" -exec "$PY_ENV_DIR/bin/pylint" "$@" diff --git a/bin/impala-python b/bin/impala-python deleted file mode 100755 index 1fcdd170c..000000000 --- a/bin/impala-python +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -############################################################################## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -############################################################################## - -source "$(dirname "$0")/impala-python-common.sh" -exec "$PY_ENV_DIR/bin/python" "$@" diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh deleted file mode 100644 index 4026029fa..000000000 --- a/bin/impala-python-common.sh +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This file is intended to be sourced to perform common setup for -# $IMPALA_HOME/bin/impala-py* executables. - -set -euo pipefail -. $IMPALA_HOME/bin/report_build_error.sh -setup_report_build_error - -. $IMPALA_HOME/bin/set-pythonpath.sh - -export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ - --print-ld-library-path)" - -PY_DIR="$(dirname "$0")/../infra/python" -PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}" -python "$PY_DIR/bootstrap_virtualenv.py" diff --git a/bin/impala-python3-common.sh b/bin/impala-python3-common.sh index 06bf3a87a..c6cca8737 100644 --- a/bin/impala-python3-common.sh +++ b/bin/impala-python3-common.sh @@ -29,4 +29,4 @@ export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv PY_DIR="$(dirname "$0")/../infra/python" PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}-py3" -python "$PY_DIR/bootstrap_virtualenv.py" --python3 +python3 "$PY_DIR/bootstrap_virtualenv.py" --python3 diff --git a/bin/impala-shell.sh b/bin/impala-shell.sh index 2fff437d9..b2b664add 100755 --- a/bin/impala-shell.sh +++ b/bin/impala-shell.sh @@ -28,7 +28,7 @@ PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/bin PYTHONPATH=${PYTHONPATH}:${SHELL_HOME} export LD_LIBRARY_PATH=":$(PYTHONPATH=${PYTHONPATH} \ - python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ + python3 "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \ --print-ld-library-path)" IMPALA_PY_DIR="$(dirname "$0")/../infra/python" @@ -36,15 +36,13 @@ IMPALA_PY3_ENV_DIR="${IMPALA_PY_DIR}/env-gcc${IMPALA_GCC_VERSION}-py3" # Allow overriding the python executable IMPALA_PYTHON_EXECUTABLE="${IMPALA_PYTHON_EXECUTABLE:-${IMPALA_PY3_ENV_DIR}/bin/python3}" -# Note that this uses the external system python executable. -# IMPALA-14620: At some point, we need to explicitly invoke python3 instead of python -# here. This is not an issue if python-is-python3 is installed. -PYTHONPATH=${PYTHONPATH} python "${IMPALA_PY_DIR}/bootstrap_virtualenv.py" --python3 +# Note that this uses the external system python3 executable. +PYTHONPATH=${PYTHONPATH} python3 "${IMPALA_PY_DIR}/bootstrap_virtualenv.py" --python3 # Enable remote debugging if port was specified via environment variable if [[ ${IMPALA_SHELL_DEBUG_PORT:-0} -ne 0 ]]; then echo "installing debugpy if needed" - ${IMPALA_PY_ENV_DIR}/bin/pip install debugpy + ${IMPALA_PY3_ENV_DIR}/bin/pip install debugpy echo "impala python shell waiting for remote debugging connection on port" \ "${IMPALA_SHELL_DEBUG_PORT}" EXTRA_ARGS=" -m debugpy --listen ${IMPALA_SHELL_DEBUG_PORT} --wait-for-client" diff --git a/bin/impala-virtualenv b/bin/impala-virtualenv deleted file mode 100755 index 8f4e58f5d..000000000 --- a/bin/impala-virtualenv +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -source "$(dirname "$0")/impala-python-common.sh" -exec "$PY_ENV_DIR/bin/python" "$PY_ENV_DIR/bin/virtualenv" "$@" diff --git a/bin/init-impala-python.sh b/bin/init-impala-python.sh index 35c90b90c..198bd4475 100755 --- a/bin/init-impala-python.sh +++ b/bin/init-impala-python.sh @@ -27,29 +27,5 @@ bin=`dirname "$0"` bin=`cd "$bin"; pwd` . "$bin"/impala-config.sh > /dev/null 2>&1 -function print_usage { - echo "init-impala-python.sh - Script called from CMake to init python venvs" - echo "[-python3] : Init the python3 virtualenv (default is python2)" -} - -IS_PYTHON3=false -while [ -n "$*" ] -do - case "$1" in - -python3) - IS_PYTHON3=true - ;; - -help|*) - print_usage - exit 1 - ;; - esac - shift -done - cd $IMPALA_HOME -if $IS_PYTHON3 ; then - bin/impala-python3 -c 'print("Initialized impala-python3")' -else - bin/impala-python -c 'print("Initialized impala-python")' -fi +bin/impala-python3 -c 'print("Initialized impala-python3")' diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py index 6587bfb68..2dce0ef63 100644 --- a/infra/python/bootstrap_virtualenv.py +++ b/infra/python/bootstrap_virtualenv.py @@ -58,17 +58,11 @@ GCC_VERSION = os.environ["IMPALA_GCC_VERSION"] IMPALA_HOME = os.environ["IMPALA_HOME"] DEPS_DIR = os.path.join(os.path.dirname(__file__), "deps") -ENV_DIR_PY2 = os.path.join(os.path.dirname(__file__), - "env-gcc{0}".format(GCC_VERSION)) ENV_DIR_PY3 = os.path.join(os.path.dirname(__file__), "env-gcc{0}-py3".format(GCC_VERSION)) # Setuptools requirements file. Setuptools is required during pip install for -# some packages. Newer setuptools dropped python 2 support, and some python -# install tools don't understand that they need to get a version that works -# with the current python version. This can cause them to try to install the newer -# setuptools that won't work on python 2. Doing this as a separate step makes it -# easy to pin the version of setuptools to a Python 2 compatible version. +# some packages. SETUPTOOLS_REQS_PATH = os.path.join(DEPS_DIR, "setuptools-requirements.txt") # Requirements file with packages we need for our build and tests, which depends @@ -88,12 +82,6 @@ ADLS_REQS_PATH = os.path.join(DEPS_DIR, "adls-requirements.txt") # is enabled. GCOVR_REQS_PATH = os.path.join(DEPS_DIR, "gcovr-requirements.txt") -# Extra packages specific to python 3 -PY3_REQS_PATH = os.path.join(DEPS_DIR, "py3-requirements.txt") - -# Extra packages specific to python 2 -PY2_REQS_PATH = os.path.join(DEPS_DIR, "py2-requirements.txt") - def delete_virtualenv_if_exist(venv_dir): if os.path.exists(venv_dir): @@ -116,33 +104,12 @@ def detect_virtualenv_version(): return None -def create_virtualenv(venv_dir, is_py3): - if is_py3: - # Python 3 is much simpler, because there is a builtin venv command - LOG.info("Creating python3 virtualenv") - python_cmd = download_toolchain_python(is_py3) - exec_cmd([python_cmd, "-m" "venv", venv_dir]) - return - - # Python 2 - LOG.info("Creating python2 virtualenv") - build_dir = tempfile.mkdtemp() - # Try to find the virtualenv version by parsing the requirements file - # Default to "*" if we can't figure it out. - virtualenv_version = detect_virtualenv_version() - if virtualenv_version is None: - virtualenv_version = "*" - # Open the virtualenv tarball - virtualenv_tarball = \ - find_file(DEPS_DIR, "virtualenv-{0}.tar.gz".format(virtualenv_version)) - file = tarfile.open(virtualenv_tarball, "r:gz") - for member in file.getmembers(): - file.extract(member, build_dir) - file.close() - python_cmd = download_toolchain_python(is_py3) - exec_cmd([python_cmd, find_file(build_dir, "virtualenv*", "virtualenv.py"), "--quiet", - "--python", python_cmd, venv_dir]) - shutil.rmtree(build_dir) +def create_virtualenv(venv_dir): + # Use Python 3's builtin venv command + LOG.info("Creating python3 virtualenv") + python_cmd = download_toolchain_python() + exec_cmd([python_cmd, "-m" "venv", venv_dir]) + return def exec_cmd(args, **kwargs): @@ -172,7 +139,7 @@ def select_cc(): return cc -def exec_pip_install(venv_dir, is_py3, args, cc="no-cc-available", env=None): +def exec_pip_install(venv_dir, args, cc="no-cc-available", env=None): '''Executes "pip install" with the provided command line arguments. If 'cc' is set, it is used as the C compiler. Otherwise compilation of C/C++ code is disabled by setting the CC environment variable to a bogus value. @@ -201,12 +168,8 @@ def exec_pip_install(venv_dir, is_py3, args, cc="no-cc-available", env=None): # Don't call the virtualenv pip directly, it uses a hashbang to to call the python # virtualenv using an absolute path. If the path to the virtualenv is very long, the # hashbang won't work. - if is_py3: - impala_pip_base_cmd = [os.path.join(venv_dir, "bin", "python3"), - os.path.join(venv_dir, "bin", "pip3"), "install", "-v"] - else: - impala_pip_base_cmd = [os.path.join(venv_dir, "bin", "python"), - os.path.join(venv_dir, "bin", "pip"), "install", "-v"] + impala_pip_base_cmd = [os.path.join(venv_dir, "bin", "python3"), + os.path.join(venv_dir, "bin", "pip3"), "install", "-v"] # Passes --no-binary for IMPALA-3767: without this, Cython (and # several other packages) fail download. @@ -216,18 +179,11 @@ def exec_pip_install(venv_dir, is_py3, args, cc="no-cc-available", env=None): third_party_pkg_install_cmd = \ impala_pip_base_cmd[:] + ["--no-binary", ":all:", "--no-cache-dir"] - # When using a custom mirror, we also must use the index of that mirror. - # The python 3 virtualenv has trouble with using --index-url with PYPI_MIRROR, - # so it falls back to --no-index, which works fine. - if "PYPI_MIRROR" in os.environ and not is_py3: - third_party_pkg_install_cmd.extend(["--index-url", - "%s/simple" % os.environ["PYPI_MIRROR"]]) - else: - # Prevent fetching additional packages from the index. If we forget to add a package - # to one of the requirements.txt files, this should trigger an error. However, we will - # still access the index for version/dependency resolution, hence we need to change it - # when using a private mirror. - third_party_pkg_install_cmd.append("--no-index") + # Prevent fetching additional packages from the index. If we forget to add a package + # to one of the requirements.txt files, this should trigger an error. However, we will + # still access the index for version/dependency resolution, hence we need to change it + # when using a private mirror. + third_party_pkg_install_cmd.append("--no-index") third_party_pkg_install_cmd.extend(["--find-links", "file://%s" % pathname2url(os.path.abspath(DEPS_DIR))]) @@ -255,7 +211,7 @@ def find_file(*paths): return files[0] -def download_toolchain_python(is_py3): +def download_toolchain_python(): '''Grabs the Python implementation from the Impala toolchain, using the machinery from bin/bootstrap_toolchain.py. Skip the download if SKIP_TOOLCHAIN_BOOTSTRAP=true in the environment. In that case @@ -267,36 +223,27 @@ def download_toolchain_python(is_py3): raise Exception("Impala environment not set up correctly, make sure " "$IMPALA_TOOLCHAIN_PACKAGES_HOME is set.") - if is_py3: - package = ToolchainPackage("python", - explicit_version=os.environ["IMPALA_PYTHON3_VERSION"]) - else: - package = ToolchainPackage("python") + package = ToolchainPackage("python") if package.needs_download() and \ not (os.environ.get(SKIP_TOOLCHAIN_BOOTSTRAP) == 'true'): package.download() - if is_py3: - python_cmd = os.path.join(package.pkg_directory(), "bin/python3") - else: - python_cmd = os.path.join(package.pkg_directory(), "bin/python") + python_cmd = os.path.join(package.pkg_directory(), "bin/python3") if not os.path.exists(python_cmd): raise Exception("Unexpected error bootstrapping python from toolchain: {0} does not " "exist".format(python_cmd)) return python_cmd -def install_deps(venv_dir, is_py3): - py_str = "3" if is_py3 else "2" - LOG.info("Installing setuptools into the python{0} virtualenv".format(py_str)) - exec_pip_install(venv_dir, is_py3, ["-r", SETUPTOOLS_REQS_PATH]) - if is_py3: - exec_pip_install(venv_dir, is_py3, ["packaging==24.1"]) +def install_deps(venv_dir): + LOG.info("Installing setuptools into the python3 virtualenv") + exec_pip_install(venv_dir, ["-r", SETUPTOOLS_REQS_PATH]) + exec_pip_install(venv_dir, ["packaging==24.1"]) cc = select_cc() if cc is None: raise Exception("CC not available") env = dict(os.environ) - LOG.info("Installing packages into the python{0} virtualenv".format(py_str)) - exec_pip_install(venv_dir, is_py3, ["-r", REQS_PATH], cc=cc, env=env) + LOG.info("Installing packages into the python3 virtualenv") + exec_pip_install(venv_dir, ["-r", REQS_PATH], cc=cc, env=env) mark_reqs_installed(venv_dir, REQS_PATH) @@ -312,7 +259,7 @@ def toolchain_pkg_dir(pkg_name): pkg_name + "-" + pkg_version) -def install_adls_deps(venv_dir, is_py3): +def install_adls_deps(venv_dir): # The ADLS dependencies require that the OS is at least CentOS 6.7 or above, # which is why we break this into a seperate step. If the target filesystem is # ADLS, the expectation is that the dev environment is running at least CentOS 6.7. @@ -322,16 +269,12 @@ def install_adls_deps(venv_dir, is_py3): return True cc = select_cc() assert cc is not None - py_str = "3" if is_py3 else "2" - LOG.info("Installing ADLS packages into the python{0} virtualenv".format(py_str)) - exec_pip_install(venv_dir, is_py3, ["-r", ADLS_REQS_PATH], cc=cc) + LOG.info("Installing ADLS packages into the python3 virtualenv") + exec_pip_install(venv_dir, ["-r", ADLS_REQS_PATH], cc=cc) mark_reqs_installed(venv_dir, ADLS_REQS_PATH) -def install_gcovr_deps(venv_dir, is_py3): - # Gcovr is only installed in the python3 virtualenv - if not is_py3: - return +def install_gcovr_deps(venv_dir): if not reqs_are_installed(venv_dir, GCOVR_REQS_PATH): # Gcovr takes several minutes to install, so we only install it if this is a coverage # build. We detect a coverage build by reading ${IMPALA_HOME}/.cmake_build_type. @@ -353,28 +296,11 @@ def install_gcovr_deps(venv_dir, is_py3): cc = select_cc() assert cc is not None LOG.info("Installing gcovr packages into the python3 virtualenv") - exec_pip_install(venv_dir, is_py3, ["-r", GCOVR_REQS_PATH], cc=cc) + exec_pip_install(venv_dir, ["-r", GCOVR_REQS_PATH], cc=cc) mark_reqs_installed(venv_dir, GCOVR_REQS_PATH) -def install_py_version_deps(venv_dir, is_py3): - cc = select_cc() - assert cc is not None - if not is_py3: - if not reqs_are_installed(venv_dir, PY2_REQS_PATH): - # These are extra python2-only packages - LOG.info("Installing python2 packages into the virtualenv") - exec_pip_install(venv_dir, is_py3, ["-r", PY2_REQS_PATH], cc=cc) - mark_reqs_installed(venv_dir, PY2_REQS_PATH) - else: - if not reqs_are_installed(venv_dir, PY3_REQS_PATH): - # These are extra python3-only packages - LOG.info("Installing python3 packages into the virtualenv") - exec_pip_install(venv_dir, is_py3, ["-r", PY3_REQS_PATH], cc=cc) - mark_reqs_installed(venv_dir, PY3_REQS_PATH) - - -def install_kudu_client_if_possible(venv_dir, is_py3): +def install_kudu_client_if_possible(venv_dir): '''Installs the Kudu python module if possible, which depends on the toolchain and the compiled requirements in requirements.txt. If the toolchain isn't available, nothing will be done.''' @@ -386,13 +312,11 @@ def install_kudu_client_if_possible(venv_dir, is_py3): LOG.debug("Skipping Kudu: %s doesn't exist" % kudu_base_dir) return - py_str = "3" if is_py3 else "2" - LOG.info("Installing Kudu into the python{0} virtualenv".format(py_str)) + LOG.info("Installing Kudu into the python3 virtualenv") # The installation requires that KUDU_HOME/build/latest exists. An empty directory # structure will be made to satisfy that. The Kudu client headers and lib will be made # available through GCC environment variables. - fake_kudu_build_dir = os.path.join(tempfile.gettempdir(), - "virtualenv-kudu{0}".format(py_str)) + fake_kudu_build_dir = os.path.join(tempfile.gettempdir(), "virtualenv-kudu3") try: artifact_dir = os.path.join(fake_kudu_build_dir, "build", "latest") if not os.path.exists(artifact_dir): @@ -409,7 +333,7 @@ def install_kudu_client_if_possible(venv_dir, is_py3): env["CPLUS_INCLUDE_PATH"] = os.path.join(kudu_client_dir, "include") env["LIBRARY_PATH"] = os.path.pathsep.join([os.path.join(kudu_client_dir, 'lib'), os.path.join(kudu_client_dir, 'lib64')]) - exec_pip_install(venv_dir, is_py3, ["-r", KUDU_REQS_PATH], cc=cc, env=env) + exec_pip_install(venv_dir, ["-r", KUDU_REQS_PATH], cc=cc, env=env) mark_reqs_installed(venv_dir, KUDU_REQS_PATH) finally: try: @@ -478,11 +402,11 @@ def reqs_are_installed(venv_dir, reqs_path): installed_reqs_file.close() -def setup_virtualenv_if_not_exists(venv_dir, is_py3): +def setup_virtualenv_if_not_exists(venv_dir): if not (reqs_are_installed(venv_dir, REQS_PATH)): delete_virtualenv_if_exist(venv_dir) - create_virtualenv(venv_dir, is_py3) - install_deps(venv_dir, is_py3) + create_virtualenv(venv_dir) + install_deps(venv_dir) LOG.debug("Virtualenv setup complete") @@ -511,17 +435,13 @@ if __name__ == "__main__": logging.basicConfig(level=getattr(logging, options.log_level)) - if options.python3: - venv_dir = ENV_DIR_PY3 - else: - venv_dir = ENV_DIR_PY2 + venv_dir = ENV_DIR_PY3 if options.rebuild: delete_virtualenv_if_exist(venv_dir) # Complete as many bootstrap steps as possible (see file comment for the steps). - setup_virtualenv_if_not_exists(venv_dir, options.python3) - install_kudu_client_if_possible(venv_dir, options.python3) - install_adls_deps(venv_dir, options.python3) - install_py_version_deps(venv_dir, options.python3) - install_gcovr_deps(venv_dir, options.python3) + setup_virtualenv_if_not_exists(venv_dir) + install_kudu_client_if_possible(venv_dir) + install_adls_deps(venv_dir) + install_gcovr_deps(venv_dir) diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py index c32cf1fe0..64a86d578 100755 --- a/infra/python/deps/pip_download.py +++ b/infra/python/deps/pip_download.py @@ -40,7 +40,6 @@ PYPI_MIRROR = os.environ.get('PYPI_MIRROR', 'https://pypi.python.org') # The requirement files that list all of the required packages and versions. REQUIREMENTS_FILES = ['requirements.txt', 'setuptools-requirements.txt', 'kudu-requirements.txt', 'adls-requirements.txt', - 'py2-requirements.txt', 'py3-requirements.txt', 'gcovr-requirements.txt'] diff --git a/infra/python/deps/py2-requirements.txt b/infra/python/deps/py2-requirements.txt deleted file mode 100644 index fa0db244c..000000000 --- a/infra/python/deps/py2-requirements.txt +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Python2-only requirements - -flake8 == 3.9.2 - mccabe == 0.6.1 - pycodestyle == 2.7.0 - pyflakes == 2.3.1 - enum34 == 1.1.10 - typing == 3.10.0.0 - configparser == 4.0.2 - functools32 == 3.2.3-2 - importlib-metadata == 2.1.3 - contextlib2 == 0.6.0 - pathlib2 == 2.3.7.post1 - zipp == 1.2.0 -k5test==0.9.2 -prettytable == 0.7.2 diff --git a/infra/python/deps/py3-requirements.txt b/infra/python/deps/py3-requirements.txt deleted file mode 100644 index ec5e6be7f..000000000 --- a/infra/python/deps/py3-requirements.txt +++ /dev/null @@ -1,51 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Python3-only requirements - -prettytable == 2.5.0 - wcwidth == 0.2.14 -pylint == 2.10.2 - astroid == 2.7.3 - lazy-object-proxy == 1.6.0 - wrapt == 1.12.1 - typed-ast == 1.4.3 - configparser == 4.0.2 - isort == 5.13.2 - futures == 3.3.0; python_version == "2.7" - poetry-core == 1.9.1 - singledispatch == 3.6.1 - toml == 0.10.2 - platformdirs == 2.4.1 - typing-extensions == 3.10.0.2 -k5test==0.10.3 -kazoo==2.8.0 - -# Pytest 6.2.5 and its dependents -pytest == 6.2.5 - attrs == 19.2.0 - iniconfig == 1.1.1 - pluggy == 0.12.0 - py == 1.8.2 - pytest-forked == 1.6.0 - pytest-reportlog == 0.4.0 - pytest-timeout == 2.2.0 - pytest-xdist == 2.4.0 - -# Below are needed only for dev -flake8==3.9.2 -flake8-unused-arguments==0.0.13 diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt index 25d89a338..eb0c8f575 100644 --- a/infra/python/deps/requirements.txt +++ b/infra/python/deps/requirements.txt @@ -22,6 +22,11 @@ allpairspy == 2.5.0 argparse == 1.4.0 +flake8==3.9.2 + mccabe == 0.6.1 + pyflakes==2.3.1 + pycodestyle==2.7.0 +flake8-unused-arguments==0.0.13 future == 0.18.3 hdfs == 2.0.2 docopt == 0.6.2 @@ -31,12 +36,28 @@ impyla == 0.21a3 pure-sasl == 0.6.2 # six == 1.14.0 (specified separately) thrift_sasl == 0.4.3 +isort == 5.13.2 + poetry-core == 1.9.1 +k5test==0.10.3 +kazoo==2.8.0 kerberos == 1.3.1 pexpect == 3.3 pg8000 == 1.10.2 +prettytable == 2.5.0 + wcwidth == 0.2.14 prometheus-client == 0.12.0 psutil == 5.6.3 pyparsing == 2.4.7 +pytest == 6.2.5 + attrs == 19.2.0 + iniconfig == 1.1.1 + pluggy == 0.13.1 + py == 1.8.2 + pytest-forked == 1.6.0 + pytest-reportlog == 0.4.0 + pytest-timeout == 2.2.0 + pytest-xdist == 2.4.0 + toml == 0.10.2 python-magic == 0.4.11 pywebhdfs == 0.3.2 pbr == 3.1.1 @@ -49,7 +70,6 @@ sasl == 0.2.1 six == 1.14.0 sqlparse == 0.3.1 texttable == 0.8.3 -virtualenv == 16.7.10 avro==1.10.2 thrift==0.16.0 distro==1.6.0 diff --git a/shell/CMakeLists.txt b/shell/CMakeLists.txt index c60afae1b..d20daf72f 100644 --- a/shell/CMakeLists.txt +++ b/shell/CMakeLists.txt @@ -25,13 +25,6 @@ set(PIP_LOC "~/.cache/impala_pip") # Tests depend on installing system pythons to specific locations, so we error if they # won't match what's expected in this config and make_shell_tarball.sh. set(PYTHON_EXES $ENV{IMPALA_EXTRA_PACKAGE_PYTHONS}) -if (NOT $ENV{IMPALA_SYSTEM_PYTHON2} STREQUAL "") - get_filename_component(PYTHON_NAME $ENV{IMPALA_SYSTEM_PYTHON2} NAME) - if (NOT ${PYTHON_NAME} STREQUAL "python2") - message(FATAL_ERROR "IMPALA_SYSTEM_PYTHON2 must be a binary named python2") - endif() - list(APPEND PYTHON_EXES $ENV{IMPALA_SYSTEM_PYTHON2}) -endif() if (NOT $ENV{IMPALA_SYSTEM_PYTHON3} STREQUAL "") get_filename_component(PYTHON_NAME $ENV{IMPALA_SYSTEM_PYTHON3} NAME) if (NOT ${PYTHON_NAME} STREQUAL "python3") @@ -60,9 +53,6 @@ foreach(PYTHON_EXE IN LISTS PYTHON_EXES) # different distributions add_custom_target(${PYTHON_NAME}_venv BYPRODUCTS "${VENV}" - # IMPALA-14606: create_virtualenv.sh may invoke impala-virtualenv, which require - # python2_venv. Only uncomment following line if building python2_venv. - # DEPENDS impala_python COMMAND "${CMAKE_SOURCE_DIR}/bin/cmake_aux/create_virtualenv.sh" "${PYTHON_EXE}" "${VENV}" COMMAND "${VENV}/bin/pip" install --cache-dir "${PIP_CACHE}" --upgrade pip @@ -97,13 +87,6 @@ add_custom_target(shell_pypi_test_package "${CMAKE_SOURCE_DIR}/shell/packaging/make_python_package.sh" ) -# Tests expect to find venvs at 'python2_venv' and 'python3_venv' in tests/shell/util.py. -# IMPALA-14606: Stop building 'python2_venv' and always run with IMPALA_USE_PYTHON3_TESTS=true. -# set(PYTHON2_VENV "${VENV_LOC}/python2_venv") -# add_custom_target(shell_python2_install DEPENDS python2_venv shell_pypi_test_package -# COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python2" "${SHELL_TEST_PKG}" -# ) - set(PYTHON3_VENV "${VENV_LOC}/python3_venv") add_custom_target(shell_python3_install DEPENDS python3_venv shell_pypi_test_package COMMAND "${PYTHON3_VENV}/bin/pip" install --cache-dir "${PIP_LOC}/python3" "${SHELL_TEST_PKG}" diff --git a/shell/packaging/make_shell_tarball.sh b/shell/packaging/make_shell_tarball.sh index acbc8436c..41a548570 100755 --- a/shell/packaging/make_shell_tarball.sh +++ b/shell/packaging/make_shell_tarball.sh @@ -56,15 +56,11 @@ for PYTHON_EXE in $*; do echo "Deleting all files in ${TARBALL_ROOT}/install_py${PYTHON_VERSION}" rm -rf ${TARBALL_ROOT}/install_py${PYTHON_VERSION} 2>&1 > /dev/null echo "Installing for python ${PYTHON_VERSION}" - # Use pip that matches the major version - if [[ $PYTHON_MAJOR_VERSION == 2 ]]; then - source ${IMPALA_HOME}/shell/build/python2_venv/bin/activate + # Use pip from a matching python version + if [[ $IMPALA_SYSTEM_PYTHON3_VERSION == $PYTHON_VERSION ]]; then + source ${IMPALA_HOME}/shell/build/python3_venv/bin/activate else - if [[ $IMPALA_SYSTEM_PYTHON3_VERSION == $PYTHON_VERSION ]]; then - source ${IMPALA_HOME}/shell/build/python3_venv/bin/activate - else - source ${IMPALA_HOME}/shell/build/python${PYTHON_VERSION}_venv/bin/activate - fi + source ${IMPALA_HOME}/shell/build/python${PYTHON_VERSION}_venv/bin/activate fi mkdir -p ${TARBALL_ROOT}/install_py${PYTHON_VERSION} diff --git a/tests/common/network.py b/tests/common/network.py index ccbb1fcbb..924307cf0 100644 --- a/tests/common/network.py +++ b/tests/common/network.py @@ -64,12 +64,6 @@ CERT_TO_CA_MAP = { } REQUIRED_MIN_OPENSSL_VERSION = 0x10001000 -# Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5 -# with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already -if IS_REDHAT_DERIVATIVE: - REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2, 7, 5) -else: - REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2, 7, 9) _openssl_version_number = getattr(ssl, "OPENSSL_VERSION_NUMBER", None) if _openssl_version_number is None: SKIP_SSL_MSG = "Legacy OpenSSL module detected" diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py index 7cc9e2b9d..ff82f2c9f 100644 --- a/tests/custom_cluster/test_client_ssl.py +++ b/tests/custom_cluster/test_client_ssl.py @@ -31,7 +31,7 @@ import time from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.impala_service import ImpaladService -from tests.common.network import SKIP_SSL_MSG, REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 +from tests.common.network import SKIP_SSL_MSG from tests.common.test_dimensions import create_client_protocol_dimension from tests.common.test_vector import BEESWAX from tests.shell.util import run_impala_shell_cmd, run_impala_shell_cmd_no_expect, \ @@ -79,12 +79,6 @@ class TestClientSsl(CustomClusterTestSuite): "--hostname=localhost " # Required to match hostname in certificate ).format(CERT_DIR) - @classmethod - def setup_class(cls): - if sys.version_info < REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12: - pytest.skip("Python version does not support tls 1.2") - super(TestClientSsl, cls).setup_class() - @pytest.mark.execute_serially @CustomClusterTestSuite.with_args(impalad_args=SSL_ARGS, statestored_args=SSL_ARGS, catalogd_args=SSL_ARGS) @@ -141,8 +135,7 @@ class TestClientSsl(CustomClusterTestSuite): def add_test_dimensions(cls): super(TestClientSsl, cls).add_test_dimensions() # Limit the test dimensions to avoid long run times. This runs hs2 and hs2-http - # with only the dev shells (python2 and python3) for a total of up to 4 - # dimensions. + # with only the python3 dev shell (2 total dimensions). cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension()) cls.ImpalaTestMatrix.add_dimension( create_impala_shell_executable_dimension(dev_only=True)) @@ -291,28 +284,3 @@ class TestClientSsl(CustomClusterTestSuite): response = requests.get(url, verify="%s/server-cert.pem" % CERT_DIR) assert response.status_code == requests.codes.ok, url - -# Run when the python version is too low to support TLS 1.2, to check that impala-shell -# returns the expected warning. -class TestClientSslUnsupported(CustomClusterTestSuite): - @classmethod - def setup_class(cls): - if sys.version_info >= REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12: - pytest.skip("This test is only run with older versions of python") - super(TestClientSslUnsupported, cls).setup_class() - - SSL_ARGS = ("--ssl_client_ca_certificate=%s/server-cert.pem " - "--ssl_server_certificate=%s/server-cert.pem " - "--ssl_private_key=%s/server-key.pem " - "--hostname=localhost " # Required to match hostname in certificate - % (CERT_DIR, CERT_DIR, CERT_DIR)) - - @pytest.mark.execute_serially - @CustomClusterTestSuite.with_args(impalad_args=SSL_ARGS, - statestored_args=SSL_ARGS, - catalogd_args=SSL_ARGS) - @pytest.mark.skipif(SKIP_SSL_MSG != "", reason=SKIP_SSL_MSG) - def test_shell_warning(self, vector): - result = run_impala_shell_cmd_no_expect(vector, ["--ssl", "-q", "select 1 + 2"]) - assert "Warning: TLSv1.2 is not supported for Python < 2.7.9" in result.stderr, \ - result.stderr diff --git a/tests/custom_cluster/test_thrift_socket.py b/tests/custom_cluster/test_thrift_socket.py index 8097ac709..120a1df6f 100644 --- a/tests/custom_cluster/test_thrift_socket.py +++ b/tests/custom_cluster/test_thrift_socket.py @@ -35,12 +35,6 @@ from tests.common.test_vector import ImpalaTestVector from tests.shell.util import ImpalaShell REQUIRED_MIN_OPENSSL_VERSION = 0x10001000 -# Python supports TLSv1.2 from 2.7.9 officially but on Red Hat/CentOS Python2.7.5 -# with newer python-libs (eg python-libs-2.7.5-77) supports TLSv1.2 already -if IS_REDHAT_DERIVATIVE: - REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2, 7, 5) -else: - REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12 = (2, 7, 9) _openssl_version_number = getattr(ssl, "OPENSSL_VERSION_NUMBER", None) if _openssl_version_number is None: SKIP_SSL_MSG = "Legacy OpenSSL module detected" @@ -66,12 +60,6 @@ IDLE_ARGS = " --idle_client_poll_period_s=1 -v=2" class TestThriftSocket(CustomClusterTestSuite): """ Check if thrift timeout errors are detected properly """ - @classmethod - def setup_class(cls): - if sys.version_info < REQUIRED_MIN_PYTHON_VERSION_FOR_TLSV12: - pytest.skip("Python version does not support tls 1.2") - super(TestThriftSocket, cls).setup_class() - @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args=IDLE_ARGS, cluster_size=1, disable_log_buffering=True) diff --git a/tests/run-tests.py b/tests/run-tests.py index 90e4a1855..a5608617b 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env impala-env-versioned-python +#!/usr/bin/env impala-python3 # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py index 1f9c454b6..1443a9aa6 100644 --- a/tests/shell/test_shell_commandline.py +++ b/tests/shell/test_shell_commandline.py @@ -133,9 +133,6 @@ def tmp_file(): class TestImpalaShell(ImpalaTestSuite): """A set of sanity tests for the Impala shell commandline parameters. - - The tests need to maintain Python 2.4 compatibility as a sub-goal of having - shell tests is to ensure that it's not broken in systems running Python 2.4. The tests need a running impalad instance in order to execute queries. TODO: diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py index f6bd7c49f..b1c7bea31 100755 --- a/tests/shell/test_shell_interactive.py +++ b/tests/shell/test_shell_interactive.py @@ -421,8 +421,6 @@ class TestImpalaShellInteractive(ImpalaTestSuite): def test_sigusr1_stacktraces(self, vector): if vector.get_value('strict_hs2_protocol'): pytest.skip("Strict HS2 mode doesn't support sleep() function") - if vector.get_value('impala_shell') in ['dev', 'python2']: - pytest.skip("Python 2 doesn't support faulthandler") command = "select sleep(5000); quit;" p = ImpalaShell(vector) p.send_cmd(command) diff --git a/tests/shell/util.py b/tests/shell/util.py index 455b0ad98..1aaae6c9a 100755 --- a/tests/shell/util.py +++ b/tests/shell/util.py @@ -378,14 +378,9 @@ def get_dev_impala_shell_executable(): def create_impala_shell_executable_dimension(dev_only=False): _, include_pypi = get_dev_impala_shell_executable() dimensions = [] - python3_pytest = (os.getenv("IMPALA_USE_PYTHON3_TESTS", "true") == "true") assert os.getenv("IMPALA_SYSTEM_PYTHON3"), "Must set env var IMPALA_SYSTEM_PYTHON3!" dimensions.append('dev3') - if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest: - dimensions.append('dev') if include_pypi and not dev_only: - if os.getenv("IMPALA_SYSTEM_PYTHON2") and not python3_pytest: - dimensions.append('python2') if os.getenv("IMPALA_SYSTEM_PYTHON3"): dimensions.append('python3') return ImpalaTestDimension('impala_shell', *dimensions) @@ -396,11 +391,8 @@ def get_impala_shell_executable(vector): # use 'dev3' as the default. impala_shell_executable, _ = get_dev_impala_shell_executable() executable_map = { - 'dev': ['env', 'IMPALA_PYTHON_EXECUTABLE=python', - 'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable], 'dev3': ['env', 'IMPALA_PYTHON_EXECUTABLE=python3', 'IMPALA_SHELL_PYTHON_FALLBACK=false', impala_shell_executable], - 'python2': [os.path.join(IMPALA_HOME, 'shell/build/python2_venv/bin/impala-shell')], 'python3': [os.path.join(IMPALA_HOME, 'shell/build/python3_venv/bin/impala-shell')] } return executable_map[vector.get_value_with_default('impala_shell', 'dev3')]
