Repository: incubator-impala Updated Branches: refs/heads/master 014c5603f -> bad10da4a
IMPALA-5375: Builds on CentOS 6.4 failing with broken python dependencies Builds on CentOS 6.4 fail due to dependencies not met for the new 'cryptography' python package. The ADLS commit states that the new packages are only required for ADLS and that ADLS on a dev environment is only supported from CentOS 6.7. This patch moves the compiled requirements for ADLS from compiled-requirements.txt to adls-requirements.txt and passing a compiler to the Pip environment while installing the ADLS requirements. Testing: Tested it on a machine that with TARGET_FILESYSTEM='adls' and also tested it on a CentOS 6.4 machine with the default configuration. Change-Id: I7d456a861a85edfcad55236aa8b0dbac2ff6fc78 Reviewed-on: http://gerrit.cloudera.org:8080/6998 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4e549799 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4e549799 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4e549799 Branch: refs/heads/master Commit: 4e5497995b3b1c4e8db658a3a54ab458c5430572 Parents: 014c560 Author: Sailesh Mukil <[email protected]> Authored: Thu May 25 17:58:33 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Fri May 26 07:52:40 2017 +0000 ---------------------------------------------------------------------- infra/python/bootstrap_virtualenv.py | 14 ++++++++------ infra/python/deps/adls-requirements.txt | 5 +++++ infra/python/deps/compiled-requirements.txt | 5 ----- 3 files changed, 13 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e549799/infra/python/bootstrap_virtualenv.py ---------------------------------------------------------------------- diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py index 4fb8689..e26aaf3 100644 --- a/infra/python/bootstrap_virtualenv.py +++ b/infra/python/bootstrap_virtualenv.py @@ -218,15 +218,17 @@ def install_compiled_deps_if_possible(): return True def install_adls_deps(): - # The ADLS dependencies require that the OS is at least CentOS 6.6 or above, + # The ADLS dependencies require that the OS is at least CentOS 6.7 or above, # which is why we break this into a seperate step. If the target filesystem is - # ADLS, the expectation is that the dev environment is running at least CentOS 6.6. - if reqs_are_installed(ADLS_REQS_PATH): - LOG.debug("Skipping ADLS deps: matching adls-installed-requirements.txt found") - return True + # ADLS, the expectation is that the dev environment is running at least CentOS 6.7. if os.environ.get('TARGET_FILESYSTEM') == "adls": + if reqs_are_installed(ADLS_REQS_PATH): + LOG.debug("Skipping ADLS deps: matching adls-installed-requirements.txt found") + return True + cc = select_cc() + assert cc is not None LOG.info("Installing ADLS packages into the virtualenv") - exec_pip_install(["-r", ADLS_REQS_PATH]) + exec_pip_install(["-r", ADLS_REQS_PATH], cc=cc) mark_reqs_installed(ADLS_REQS_PATH) def install_kudu_client_if_possible(): http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e549799/infra/python/deps/adls-requirements.txt ---------------------------------------------------------------------- diff --git a/infra/python/deps/adls-requirements.txt b/infra/python/deps/adls-requirements.txt index f027a6e..9df42b6 100644 --- a/infra/python/deps/adls-requirements.txt +++ b/infra/python/deps/adls-requirements.txt @@ -18,4 +18,9 @@ # The following dependencies depend on cffi, so it must be installed after the toolchain # is bootstrapped and all requirements in requirements.txt and compiled-requirements.txt # are installed into the virtualenv. + +pycparser == 2.17 +cffi==1.10.0 +cryptography==1.8.1 + scandir == 1.5 azure-datalake-store == 0.0.9 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4e549799/infra/python/deps/compiled-requirements.txt ---------------------------------------------------------------------- diff --git a/infra/python/deps/compiled-requirements.txt b/infra/python/deps/compiled-requirements.txt index b9273d1..b3f9f4d 100644 --- a/infra/python/deps/compiled-requirements.txt +++ b/infra/python/deps/compiled-requirements.txt @@ -32,11 +32,6 @@ impyla == 0.14.0 thrift == 0.9.0 thrift_sasl == 0.1.0 psutil == 0.7.1 -# Required for ADLS Python client - pycparser == 2.17 - cffi==1.10.0 - cryptography==1.8.1 - scandir == 1.5 # Required for Kudu: Cython == 0.23.4 numpy == 1.10.4
