test-with-docker: allow built images to be used with "docker run" easily.
Configures the built container to enter into a script that starts the minicluster. As a result, "docker run -ti <container>" will launch the user into a shell with the Impala minicluster and the impala development cluster running. To handle cases where users don't specify --privileged, we skip Kudu if it NTP seems unavailable. Change-Id: Ib8d6a28d4cb4ab019cd72415024b23374a6d9e2f Reviewed-on: http://gerrit.cloudera.org:8080/11781 Reviewed-by: Philip Zeyliger <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/de0c6bd6 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/de0c6bd6 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/de0c6bd6 Branch: refs/heads/master Commit: de0c6bd6bd0db163d2820ae238bee5887f410f52 Parents: c170107 Author: Philip Zeyliger <[email protected]> Authored: Mon Oct 22 21:38:14 2018 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Fri Oct 26 18:44:58 2018 +0000 ---------------------------------------------------------------------- docker/entrypoint.sh | 65 ++++++++++++++++++++++++++++++++++------- docker/test-with-docker.py | 3 ++ 2 files changed, 58 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/de0c6bd6/docker/entrypoint.sh ---------------------------------------------------------------------- diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 1dbc6c1..50a38bd 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -145,16 +145,18 @@ function start_minicluster { # presumably because there's only one layer involved. See # https://issues.apache.org/jira/browse/KUDU-1419. set -x - pushd /home/impdev/Impala/testdata - for x in cluster/cdh*/node-*/var/lib/kudu/*/wal; do - echo $x - # This mv takes time, as it's actually copying into the latest layer. - mv $x $x-orig - mkdir $x - mv $x-orig/* $x - rmdir $x-orig - done - popd + if [ "true" = $KUDU_IS_SUPPORTED ]; then + pushd /home/impdev/Impala/testdata + for x in cluster/cdh*/node-*/var/lib/kudu/*/wal; do + echo $x + # This mv takes time, as it's actually copying into the latest layer. + mv $x $x-orig + mkdir $x + mv $x-orig/* $x + rmdir $x-orig + done + popd + fi # Wait for postgresql to really start; if it doesn't, Hive Metastore will fail to start. for i in {1..120}; do @@ -387,6 +389,42 @@ function configure_timezone() { fi } +# Exposes a shell, with the container booted with +# a minicluster. +function shell() { + echo "Starting minicluster and Impala." + # Logs is typically a symlink; remove it if so. + rm logs || true + mkdir -p logs + boot_container + impala_environment + # Kudu requires --privileged for the Docker container; see + # https://issues.apache.org/jira/browse/KUDU-2000. Because + # our goal here is convenience for new developers, we + # skip kudu if "ntptime" doesn't work, which is a good + # proxy for Kudu won't start. + if ! ntptime > /dev/null; then + export KUDU_IS_SUPPORTED=false + KUDU_MSG="Kudu is not started." + fi + start_minicluster + bin/start-impala-cluster.py + cat <<"EOF" + +========================================================== +Welcome to the Impala development environment. + +The "minicluster" is running; i.e., HDFS, HBase, Hive, +etc. are running. $KUDU_MSG + +To get started, perhaps run: + impala-shell.sh -q 'select count(*) from tpcds.web_page' +========================================================== + +EOF + exec bash +} + function main() { set -e @@ -394,6 +432,13 @@ function main() { CMD="$1" shift + # Treat shell specialy to avoid the extra logging and | cat below. + if [[ $CMD = "shell" ]]; then + shell + # shell shoud have exec'd, so if we get here, it's a failure. + exit 1 + fi + echo ">>> ${CMD} $@ (begin)" # Dump environment, for debugging env | grep -vE "AWS_(SECRET_)?ACCESS_KEY" http://git-wip-us.apache.org/repos/asf/impala/blob/de0c6bd6/docker/test-with-docker.py ---------------------------------------------------------------------- diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py index b350e4c..42808cf 100755 --- a/docker/test-with-docker.py +++ b/docker/test-with-docker.py @@ -653,6 +653,9 @@ class TestWithDocker(object): self.image = _check_output( ["docker", "commit", "-c", "LABEL pwd=" + self.git_root, + "-c", "USER impdev", + "-c", "WORKDIR /home/impdev/Impala", + "-c", 'CMD ["/home/impdev/Impala/docker/entrypoint.sh", "shell"]', container.id, "impala:built-" + self.name]).strip() logging.info("Committed docker image: %s", self.image) finally:
