This is an automated email from the ASF dual-hosted git repository. wzhou pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 3e99dfcd169fe380356eeb74064372fc38201cdc Author: Michael Smith <[email protected]> AuthorDate: Wed Nov 8 14:39:43 2023 -0800 IMPALA-12515: Clarify behavior with redundant system python Clarifies the behavior building impala-shell tarball when one of the system pythons is also included in IMPALA_EXTRA_PACKAGE_PYTHONS. System python will always replace the same version from IMPALA_EXTRA_PACKAGE_PYTHONS, as system pythons are appended to the end. Updates make_shell_tarball to delete the old ext-py install when it would be replaced rather than relying on 'pip --upgrade', and iterates by python executable first to make that possible. Change-Id: I629bdab38d98c8c4232d4cae7b0429a5118d9ff7 Reviewed-on: http://gerrit.cloudera.org:8080/20687 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- shell/make_shell_tarball.sh | 47 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/shell/make_shell_tarball.sh b/shell/make_shell_tarball.sh index 09123549b..15b44ffc4 100755 --- a/shell/make_shell_tarball.sh +++ b/shell/make_shell_tarball.sh @@ -117,39 +117,40 @@ EOF # These use the same pip caches as the virtualenvs to avoid extra downloads. This # script is a prerequisite for the pypi packaging, so there is no concurrency issue. echo "Building all external dependencies" -for MODULE in ${SHELL_HOME}/ext-py/*; do - # Sometimes there are leftover module directories from version changes. If IMPALA_HOME - # is a git repository, then we can check if the module directory is tracked by git. - # If it is not tracked, skip building it. The downside of this check is that when - # adding a new directory, it won't build until added in git. This check does not apply - # when IMPALA_HOME is not a git repository (e.g. if building from a release tarball). - if ${IS_GIT_CHECKOUT} && - ! git ls-files --error-unmatch ${MODULE} > /dev/null 2>&1 ; then - echo "WARNING: ${MODULE} is not tracked by the git repository, skipping..." - continue; - fi - pushd ${MODULE} > /dev/null 2>&1 - for PYTHON_EXE in $*; do +for PYTHON_EXE in $*; do + PYTHON_NAME=$(basename ${PYTHON_EXE}) + PYTHON_VERSION=$(${PYTHON_EXE} -c 'import sys; \ + print("{}.{}".format(sys.version_info.major, sys.version_info.minor))') + # pip install the wheel into the external dependencies directory + PIP_CACHE="~/.cache/impala_pip/${PYTHON_NAME}" + # overwrite any earlier install of the same version. System Pythons are appended to + # the list, and will always override extra definitions of the same version. + rm -rf ${TARBALL_ROOT}/ext-py${PYTHON_VERSION} + # Use the venv to get the wheel package needed for bdist_wheel below. + source ${IMPALA_HOME}/shell/build/${PYTHON_NAME}_venv/bin/activate + for MODULE in ${SHELL_HOME}/ext-py/*; do + # Sometimes there are leftover module directories from version changes. If IMPALA_HOME + # is a git repository, then we can check if the module directory is tracked by git. + # If it is not tracked, skip building it. The downside of this check is that when + # adding a new directory, it won't build until added in git. This check does not apply + # when IMPALA_HOME is not a git repository (e.g. if building from a release tarball). + if ${IS_GIT_CHECKOUT} && + ! git ls-files --error-unmatch ${MODULE} > /dev/null 2>&1 ; then + echo "WARNING: ${MODULE} is not tracked by the git repository, skipping..." + continue; + fi + pushd ${MODULE} > /dev/null 2>&1 echo "Cleaning up old build artifacts." rm -rf dist 2>&1 > /dev/null rm -rf build 2>&1 > /dev/null echo "Building ${MODULE} with ${PYTHON_EXE}" - # Use the venv to get the wheel package needed for bdist_wheel below. - PYTHON_NAME=$(basename ${PYTHON_EXE}) - source ${IMPALA_HOME}/shell/build/${PYTHON_NAME}_venv/bin/activate if [[ "$MODULE" == *"/bitarray"* ]]; then # Need to use setuptools to build wheel for bitarray module python -c "import setuptools; exec(open('setup.py').read())" -q bdist_wheel else python setup.py -q bdist_wheel clean fi - # pip install the wheel into the external dependencies directory - PIP_CACHE="~/.cache/impala_pip/${PYTHON_NAME}" - PYTHON_VERSION=$(${PYTHON_EXE} -c 'import sys; \ - print("{}.{}".format(sys.version_info.major, sys.version_info.minor))') - # Use --upgrade to suppress warnings about replacement when $* includes duplicate - # Python minor versions. - pip install --upgrade --no-deps --cache "${PIP_CACHE}" \ + pip install --no-deps --cache "${PIP_CACHE}" \ --target ${TARBALL_ROOT}/ext-py${PYTHON_VERSION} dist/*.whl done popd 2>&1 > /dev/null
