This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 11396d314 IMPALA-13384: Only install gcovr deps for coverage builds
11396d314 is described below
commit 11396d3146dfa2193420f79ec284f5212f058982
Author: Joe McDonnell <[email protected]>
AuthorDate: Tue Sep 24 17:53:12 2024 -0700
IMPALA-13384: Only install gcovr deps for coverage builds
IMPALA-13279 upgraded gcovr to 7.2 and moved it from python 2 to
python 3.8. gcovr has several dependencies that require native
compilation, and this increased the cost of initializing the
Python 3 virtualenv substantially:
Without gcovr: 1m43.279s
With gcovr and deps: 6m35.107s
This moves gcovr to its own requirements file and only installs
gcovr if this is a coverage build (detected from the
.cmake_buid_type file).
Testing:
- Verified that a coverage build does install gcovr and
produce a report
Change-Id: I1d0fd6d21273053aaf2acee39fcb83d9093d49a2
Reviewed-on: http://gerrit.cloudera.org:8080/21849
Reviewed-by: Laszlo Gaal <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
infra/python/bootstrap_virtualenv.py | 38 ++++++++++++++++++++++
...py3-requirements.txt => gcovr-requirements.txt} | 15 +--------
infra/python/deps/pip_download.py | 3 +-
infra/python/deps/py3-requirements.txt | 10 ------
4 files changed, 41 insertions(+), 25 deletions(-)
diff --git a/infra/python/bootstrap_virtualenv.py
b/infra/python/bootstrap_virtualenv.py
index f26bb84b2..ce1b8b521 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -28,6 +28,7 @@
# 2. Install most packages (including ones that require C/C++ compilation)
# 3. Install Kudu package (which uses the toolchain GCC and the installed
Cython)
# 4. Install ADLS packages if applicable
+# 5. Install GCOVR packages if this is a code coverage build
#
# This module can be run with python >= 2.7. It makes no guarantees about
usage on
# python < 2.7.
@@ -54,6 +55,8 @@ SKIP_TOOLCHAIN_BOOTSTRAP = "SKIP_TOOLCHAIN_BOOTSTRAP"
GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
+IMPALA_HOME = os.environ["IMPALA_HOME"]
+
DEPS_DIR = os.path.join(os.path.dirname(__file__), "deps")
ENV_DIR_PY2 = os.path.join(os.path.dirname(__file__),
"env-gcc{0}".format(GCC_VERSION))
@@ -80,6 +83,11 @@ KUDU_REQS_PATH = os.path.join(DEPS_DIR,
"kudu-requirements.txt")
# Interface) being installed by the requirements step.
ADLS_REQS_PATH = os.path.join(DEPS_DIR, "adls-requirements.txt")
+# Requirements for the gcovr utility. These add several minutes to
initializing the
+# virtualenv, so they are split off into their own step that only runs when
coverage
+# is enabled.
+GCOVR_REQS_PATH = os.path.join(DEPS_DIR, "gcovr-requirements.txt")
+
# Extra packages specific to python 3
PY3_REQS_PATH = os.path.join(DEPS_DIR, "py3-requirements.txt")
@@ -318,6 +326,35 @@ def install_adls_deps(venv_dir, is_py3):
mark_reqs_installed(venv_dir, ADLS_REQS_PATH)
+def install_gcovr_deps(venv_dir, is_py3):
+ # Gcovr is only installed in the python3 virtualenv
+ if not is_py3:
+ return
+ if not reqs_are_installed(venv_dir, GCOVR_REQS_PATH):
+ # Gcovr takes several minutes to install, so we only install it if this is
a coverage
+ # build. We detect a coverage build by reading
${IMPALA_HOME}/.cmake_build_type.
+ # The python virtualenv is typically initialized during the main build,
and CMake
+ # writes .cmake_build_type before the build starts. If that file doesn't
exist
+ # (usually because impala-python3 is being run manually), don't install
gcovr. Future
+ # invocations will check again and can install it if needed.
+ cmake_build_type_file = os.path.join(IMPALA_HOME, ".cmake_build_type")
+ if not os.path.isfile(cmake_build_type_file):
+ return
+ coverage_enabled = False
+ with open(cmake_build_type_file) as f:
+ for line in f:
+ if line.find("COVERAGE") != -1:
+ coverage_enabled = True
+ break
+
+ if coverage_enabled:
+ cc = select_cc()
+ assert cc is not None
+ LOG.info("Installing gcovr packages into the python3 virtualenv")
+ exec_pip_install(venv_dir, is_py3, ["-r", GCOVR_REQS_PATH], cc=cc)
+ mark_reqs_installed(venv_dir, GCOVR_REQS_PATH)
+
+
def install_py_version_deps(venv_dir, is_py3):
cc = select_cc()
assert cc is not None
@@ -485,3 +522,4 @@ if __name__ == "__main__":
install_kudu_client_if_possible(venv_dir, options.python3)
install_adls_deps(venv_dir, options.python3)
install_py_version_deps(venv_dir, options.python3)
+ install_gcovr_deps(venv_dir, options.python3)
diff --git a/infra/python/deps/py3-requirements.txt
b/infra/python/deps/gcovr-requirements.txt
similarity index 74%
copy from infra/python/deps/py3-requirements.txt
copy to infra/python/deps/gcovr-requirements.txt
index 902304489..174be6d8b 100644
--- a/infra/python/deps/py3-requirements.txt
+++ b/infra/python/deps/gcovr-requirements.txt
@@ -15,21 +15,8 @@
# specific language governing permissions and limitations
# under the License.
-# Python3-only requirements
+# Requirements for gcovr, which are only needed for coverage runs
-pylint == 2.10.2
- astroid == 2.7.3
- lazy-object-proxy == 1.6.0
- wrapt == 1.12.1
- typed-ast == 1.4.3
- configparser == 4.0.2
- isort == 4.3.21
- futures == 3.3.0; python_version == "2.7"
- singledispatch == 3.6.1
- toml == 0.10.2
- platformdirs == 2.4.1
- typing-extensions == 3.10.0.2
-k5test==0.10.3
gcovr == 7.2
Jinja2 == 3.1.4
flit-core == 3.9.0
diff --git a/infra/python/deps/pip_download.py
b/infra/python/deps/pip_download.py
index 2d965ac33..bbbc4431d 100755
--- a/infra/python/deps/pip_download.py
+++ b/infra/python/deps/pip_download.py
@@ -39,7 +39,8 @@ PYPI_MIRROR = os.environ.get('PYPI_MIRROR',
'https://pypi.python.org')
# The requirement files that list all of the required packages and versions.
REQUIREMENTS_FILES = ['requirements.txt', 'setuptools-requirements.txt',
'kudu-requirements.txt', 'adls-requirements.txt',
- 'py2-requirements.txt', 'py3-requirements.txt']
+ 'py2-requirements.txt', 'py3-requirements.txt',
+ 'gcovr-requirements.txt']
def check_digest(filename, algorithm, expected_digest):
diff --git a/infra/python/deps/py3-requirements.txt
b/infra/python/deps/py3-requirements.txt
index 902304489..b61bc461c 100644
--- a/infra/python/deps/py3-requirements.txt
+++ b/infra/python/deps/py3-requirements.txt
@@ -30,13 +30,3 @@ pylint == 2.10.2
platformdirs == 2.4.1
typing-extensions == 3.10.0.2
k5test==0.10.3
-gcovr == 7.2
- Jinja2 == 3.1.4
- flit-core == 3.9.0
- lxml == 5.2.2
- Cython == 3.0.10
- colorlog == 6.8.2
- Pygments == 2.13.0
- MarkupSafe == 2.1.5
- tomli == 2.0.1
- packaging == 24.1
