Repository: incubator-impala Updated Branches: refs/heads/master 893e6f498 -> a5ae2bfd8
IMPALA-3762: Download Python requirements before they are needed. This is needed for ASF builds. It sounds expensive, but takes less than 10 seconds if the packages are already present. Change-Id: I84103c2fb8f9a93336bf28b644ca045f15651dd6 Reviewed-on: http://gerrit.cloudera.org:8080/3452 Reviewed-by: Jim Apple <[email protected]> Tested-by: Jim Apple <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/a5ae2bfd Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/a5ae2bfd Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/a5ae2bfd Branch: refs/heads/master Commit: a5ae2bfd881ba8a39f171b0cc39c29b592015c2d Parents: 893e6f4 Author: Jim Apple <[email protected]> Authored: Tue Jun 21 13:25:50 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Wed Jun 22 14:38:57 2016 -0700 ---------------------------------------------------------------------- buildall.sh | 12 +++++++++++- infra/python/deps/download_requirements | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a5ae2bfd/buildall.sh ---------------------------------------------------------------------- diff --git a/buildall.sh b/buildall.sh index b4542d8..6944169 100755 --- a/buildall.sh +++ b/buildall.sh @@ -239,6 +239,16 @@ if [[ -z $METASTORE_SNAPSHOT_FILE && "${TARGET_FILESYSTEM}" != "hdfs" && exit 1 fi +echo "Downloading Python dependencies" +# Download all the Python dependencies we need before doing anything +# of substance. Does not re-download anything that is already present. +if ! $IMPALA_HOME/infra/python/deps/download_requirements; then + echo "Warning: Unable to download Python requirements." + echo "Warning: bootstrap_virtualenv or other Python-based tooling may fail." +else + echo "Finished downloading Python dependencies" +fi + # option to clean everything first if [ $CLEAN_ACTION -eq 1 ]; then $IMPALA_HOME/bin/clean.sh @@ -248,7 +258,7 @@ fi if [ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]; then echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap." else - echo "Downloading and extracting dependencies." + echo "Downloading and extracting toolchain dependencies." $IMPALA_HOME/bin/bootstrap_toolchain.py echo "Toolchain bootstrap complete." fi http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/a5ae2bfd/infra/python/deps/download_requirements ---------------------------------------------------------------------- diff --git a/infra/python/deps/download_requirements b/infra/python/deps/download_requirements index a7dc432..c72eabb 100755 --- a/infra/python/deps/download_requirements +++ b/infra/python/deps/download_requirements @@ -16,7 +16,7 @@ fi DIR=$(dirname "$0") # Download but don't install all packages listed in requirements.txt. -"$PIP" install --download "$DIR" -r "$DIR"/requirements.txt +"$PIP" -q install --download "$DIR" -r "$DIR"/requirements.txt # Directly download required packages not listed in requirements.txt. # For virtualenv, other scripts rely on the .tar.gz package (not a .whl package).
