Repository: incubator-impala Updated Branches: refs/heads/master fc444c102 -> ec3a1c786
download_requirements should download kudu-python and virtualenv This is required for the ASF migration, since we don't want to include all of the tarballs in the repo and we want to allow developers to build using dependencies obtained from the standard upstream sources. Also remove a workaround for an old issue with building an impyla development version package. Change-Id: Ie9216596db0f37d706ea7f77c129cecd5b070429 Reviewed-on: http://gerrit.cloudera.org:8080/3217 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Internal Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ec3a1c78 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ec3a1c78 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ec3a1c78 Branch: refs/heads/master Commit: ec3a1c7866741d0e28c6fb98e0d485422767ebbf Parents: fc444c1 Author: Tim Armstrong <[email protected]> Authored: Tue Jun 7 13:42:53 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Mon Jun 13 17:32:27 2016 -0700 ---------------------------------------------------------------------- infra/python/deps/download_requirements | 18 +++++++++++----- infra/python/deps/pip_download.py | 31 ++++++++++++++++++++++++++++ infra/python/deps/requirements.txt | 5 +++-- 3 files changed, 47 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ec3a1c78/infra/python/deps/download_requirements ---------------------------------------------------------------------- diff --git a/infra/python/deps/download_requirements b/infra/python/deps/download_requirements index 9dca8a4..fa06cd6 100755 --- a/infra/python/deps/download_requirements +++ b/infra/python/deps/download_requirements @@ -2,17 +2,25 @@ set -euo pipefail -# Prefer the virtualenv pip since this is what will actually be used during the +# Prefer the virtualenv pip and python since this is what will actually be used during the # installation and it may be a different version than the system default. VIRTUAL_ENV_PIP="$IMPALA_HOME"/infra/python/env/bin/pip if [[ -e "$VIRTUAL_ENV_PIP" ]]; then PIP="$VIRTUAL_ENV_PIP" + # Assume python is also available in the virtualenv. + PYTHON="$IMPALA_HOME"/infra/python/env/bin/python else PIP=pip + PYTHON=python fi DIR=$(dirname "$0") -# Ignore the dev version of Impyla, it can't be downloaded (it needs to be built and -# copied into the deps folder). -"$PIP" install --download "$DIR" \ - -r <(cat "$DIR"/requirements.txt | grep -v "impyla.*dev") +# Download but don't install all packages listed in requirements.txt. +"$PIP" install --download "$DIR" -r "$DIR"/requirements.txt + +# Directly download required packages not listed in requirements.txt. +# For virtualenv, other scripts rely on the .tar.gz package (not a .whl package). +# kudu-python is downloaded separately because pip install attempts to execute a +# setup.py subcommand for kudu-python that can fail even if the download succeeds. +$PYTHON $DIR/pip_download.py virtualenv 13.0.1 +$PYTHON $DIR/pip_download.py kudu-python 0.1.1 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ec3a1c78/infra/python/deps/pip_download.py ---------------------------------------------------------------------- diff --git a/infra/python/deps/pip_download.py b/infra/python/deps/pip_download.py new file mode 100755 index 0000000..34a0897 --- /dev/null +++ b/infra/python/deps/pip_download.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +# Implement the basic 'pip download' functionality in a way that gives us more control +# over which archive type is downloaded and what post-download steps are executed. +import hashlib +import json +from urllib import urlopen, URLopener +import sys + +pkg_name = sys.argv[1] +pkg_version = sys.argv[2] +pkg_type = 'sdist' # Don't download wheel archives for now +pkg_info = json.loads(urlopen('https://pypi.python.org/pypi/%s/json' % pkg_name).read()) + +found = False +downloader = URLopener() +for pkg in pkg_info['releases'][pkg_version]: + if pkg['packagetype'] == pkg_type: + print "Downloading %s from %s " % (pkg['filename'], pkg['url']) + downloader.retrieve(pkg['url'], pkg['filename']) + expected_md5 = pkg['md5_digest'] + actual_md5 = hashlib.md5(open(pkg['filename']).read()).hexdigest() + if actual_md5 != expected_md5: + print "MD5 mismatch: %s v. %s" % (expected_md5, actual_md5) + sys.exit(1) + found = True + break + +if not found: + print "Could not find archive to download for %s %s %s" % (pkg_name, pkg_version, + pkg_type) + sys.exit(1) http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ec3a1c78/infra/python/deps/requirements.txt ---------------------------------------------------------------------- diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt index 9ffafc6..b3ebfb1 100644 --- a/infra/python/deps/requirements.txt +++ b/infra/python/deps/requirements.txt @@ -1,4 +1,4 @@ -# To download requirements use the downlod_requirements script in this dir. +# To download requirements use the download_requirements script in this dir. # Remember, all modules below need to support python 2.6. @@ -61,7 +61,8 @@ texttable == 0.8.3 # The virtualenv may need to be functional even if the toolchain isn't present. The # bootstap_virtualenv.py script special-cases kudu-python, the line below is actually # functional and determines the expected kudu-python version. The version must be listed -# in the format below including # and spacing. Keep this formatting! +# in the format below including # and spacing. Keep this formatting! The kudu-python +# version in download_requirements must be kept in sync with this version. # kudu-python==0.1.1 Cython == 0.23.4 numpy == 1.10.4
