Repository: impala Updated Branches: refs/heads/master 8cbec20ea -> 1dbb3c1be
test-with-docker: add --env option to pass through env variables Adds simple mechanism to pass environment variables into test-with-docker, which is occasionally useful, especially for development and tinkering with tests. It's typically the right thing to codify the environment variables into tests, but a pass through can be handy. The implementation is simple enough, passing the variables into the docker containers. Change-Id: I03c2feda8edc2983e423f862ed210fabb845714f Reviewed-on: http://gerrit.cloudera.org:8080/11730 Reviewed-by: Laszlo Gaal <[email protected]> Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/1dbb3c1b Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/1dbb3c1b Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/1dbb3c1b Branch: refs/heads/master Commit: 1dbb3c1be634c2a3e356c3b2ee5deac2436c9815 Parents: 8cbec20 Author: Philip Zeyliger <[email protected]> Authored: Wed May 16 12:23:47 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Thu Oct 25 23:52:58 2018 +0000 ---------------------------------------------------------------------- docker/test-with-docker.py | 44 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/1dbb3c1b/docker/test-with-docker.py ---------------------------------------------------------------------- diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py index 4efd712..88f4563 100755 --- a/docker/test-with-docker.py +++ b/docker/test-with-docker.py @@ -18,6 +18,23 @@ # under the License. # +# We do not use Impala's python environment here, nor do we depend on +# non-standard python libraries to avoid needing extra build steps before +# triggering this. +import argparse +import datetime +import itertools +import logging +import multiprocessing +import multiprocessing.pool +import os +import re +import subprocess +import sys +import tempfile +import threading +import time + CLI_HELP = """\ Runs tests inside of docker containers, parallelizing different types of tests. This script first creates a docker container, checks out this repo @@ -100,22 +117,6 @@ as part of the build, in logs/docker/*/timeline.html. # or move to different suite. # - Run BE tests earlier (during data load) -# We do not use Impala's python environment here, nor do we depend on -# non-standard python libraries to avoid needing extra build steps before -# triggering this. -import argparse -import datetime -import logging -import multiprocessing -import multiprocessing.pool -import os -import re -import subprocess -import sys -import tempfile -import threading -import time - if __name__ == '__main__' and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import monitor @@ -181,6 +182,10 @@ def main(): default=os.path.expanduser("~/.ccache")) parser.add_argument('--tail', action="store_true", help="Run tail on all container log files.") + parser.add_argument('--env', metavar='K=V', default=[], action='append', + help="""Passes given environment variables (expressed as KEY=VALUE) + through containers. + """) parser.add_argument('--test', action="store_true") args = parser.parse_args() @@ -197,7 +202,8 @@ def main(): parallel_test_concurrency=args.parallel_test_concurrency, suite_concurrency=args.suite_concurrency, impalad_mem_limit_bytes=args.impalad_mem_limit_bytes, - tail=args.tail) + tail=args.tail, + env=args.env) fh = logging.FileHandler(os.path.join(_make_dir_if_not_exist(t.log_dir), "log.txt")) fh.setFormatter(logging.Formatter(LOG_FORMAT)) @@ -430,7 +436,7 @@ class TestWithDocker(object): def __init__(self, build_image, suite_names, name, cleanup_containers, cleanup_image, ccache_dir, test_mode, suite_concurrency, parallel_test_concurrency, - impalad_mem_limit_bytes, tail): + impalad_mem_limit_bytes, tail, env): self.build_image = build_image self.name = name self.containers = [] @@ -466,6 +472,7 @@ class TestWithDocker(object): self.parallel_test_concurrency = parallel_test_concurrency self.impalad_mem_limit_bytes = impalad_mem_limit_bytes self.tail = tail + self.env = env # Map suites back into objects; we ignore case for this mapping. suites = [] @@ -545,6 +552,7 @@ class TestWithDocker(object): "-v", _make_dir_if_not_exist(self.log_dir, logdir) + ":/logs", "-v", base + ":/mnt/base:ro"] + + list(itertools.chain(*[["-e", env] for env in self.env])) + extras + [image] + entrypoint).strip()
