This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit c8a21c51efee2b0dd2f416c0c91853ecc6bbf96d Author: Michael Smith <[email protected]> AuthorDate: Tue Apr 18 16:53:29 2023 -0700 IMPALA-12081: Produce multiple Java docker images This changes the docker image build code so that both Java 8 and Java 11 images can be built in the same build. Specifically, it introduces new Make targets for Java 11 docker images in addition to the regular Java 8 targets. The "docker_images" and "docker_debug_images" targets continue to behave the same way and produce Java 8 images of the same name. The "docker_java11_images" and "docker_debug_java11_images" produce the daemon docker images for Java 11. Preserves IMPALA_DOCKER_USE_JAVA11 for selecting Java 11 images when starting a cluster with container images. Change-Id: Ic2b124267c607242bc2fd6c8cd6486293a938f50 Reviewed-on: http://gerrit.cloudera.org:8080/19722 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- bin/impala-config.sh | 7 +++--- bin/jenkins/dockerized-impala-run-tests.sh | 6 ++++- bin/start-impala-cluster.py | 2 ++ docker/CMakeLists.txt | 36 +++++++++++++++++------------- docker/install_os_packages.sh | 28 +++++++++-------------- docker/setup_build_context.py | 11 +++------ 6 files changed, 44 insertions(+), 46 deletions(-) diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 3f545f284..de57cc95d 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -266,10 +266,9 @@ export IMPALA_OBS_VERSION=3.1.1-hw-42 export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"} export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"} -# When set to true, this modifies the Docker image build logic to install Java 11 -# rather than the default of Java 8. This is strictly about what is shipped in the -# Docker image, and it has no impact on what version of Java is used to compile -# Impala's Java code. +# When set to true, this instructs start-impala-cluster.py to start with Java 11 images +# created via 'make docker_java11_images' (or 'docker_debug_java11_images'). The Java +# version used in these images is independent of the Java version used to compile Impala. export IMPALA_DOCKER_USE_JAVA11=${IMPALA_DOCKER_USE_JAVA11:-"false"} # There are multiple compatible implementations of zlib. Cloudflare Zlib is an diff --git a/bin/jenkins/dockerized-impala-run-tests.sh b/bin/jenkins/dockerized-impala-run-tests.sh index 9b6ea001a..cb665b18f 100755 --- a/bin/jenkins/dockerized-impala-run-tests.sh +++ b/bin/jenkins/dockerized-impala-run-tests.sh @@ -80,7 +80,11 @@ start-impala-cluster.py --kill # Build the docker images required to start the cluster. # parquet-reader and impala-profile-tool are needed for e2e tests but not built for # non-test build. -make -j ${IMPALA_BUILD_THREADS} docker_debug_images parquet-reader impala-profile-tool +IMAGE_TYPE=docker_debug +if ${IMPALA_DOCKER_USE_JAVA11-false}; then + IMAGE_TYPE=${IMAGE_TYPE}_java11 +fi +make -j ${IMPALA_BUILD_THREADS} ${IMAGE_TYPE}_images parquet-reader impala-profile-tool source_impala_config diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py index 5a916dc46..88b886ab7 100755 --- a/bin/start-impala-cluster.py +++ b/bin/start-impala-cluster.py @@ -687,6 +687,8 @@ class DockerMiniClusterOperations(object): image_tag = daemon + "_debug" else: image_tag = daemon + if os.getenv('IMPALA_DOCKER_USE_JAVA11', '') == "true": + image_tag += "_java11" host_name = self.__gen_host_name__(daemon, instance) container_name = self.__gen_container_name__(daemon, instance) # Mount configuration into container so that we don't need to rebuild container diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt index c94e97898..7f0455760 100644 --- a/docker/CMakeLists.txt +++ b/docker/CMakeLists.txt @@ -70,27 +70,18 @@ MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}" if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") # Add a target to build a base docker image for 'build_type'. 'build_context_args' are # passed to the setup_build_context.py script. - function(add_base_image build_type build_context_args) + function(add_base_image build_type build_context_args install_os_packages_args) # Build context depends on daemons and frontend jars. # Sending the whole impala workspace including test binaries, testdata, etc # to the docker daemon can be very expensive, so we create a build context # with symlinks add_custom_target(impala_base_build_context_${build_type} COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args} + --output-dir ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type} DEPENDS daemons java ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py COMMENT "Creating impala base build context build_type=${build_type}." VERBATIM ) - # The base image Dockerfile takes an argument INSTALL_OS_PACKAGES_ARGS to specify - # arguments for install_os_packages.sh. This can can be set to "--install-debug-tools" - # to add extra utilities. Enable this for debug builds. - set(INSTALL_OS_PACKAGES_ARGS "") - if (${build_type} STREQUAL "debug") - set(INSTALL_OS_PACKAGES_ARGS "--install-debug-tools") - endif() - if ($ENV{IMPALA_DOCKER_USE_JAVA11} STREQUAL "true") - set(INSTALL_OS_PACKAGES_ARGS "${INSTALL_OS_PACKAGES_ARGS} --use-java11") - endif() # Target for the base Impala image. add_custom_target(impala_base_image_${build_type} # Run docker build inside the build context directory so that all dependencies are @@ -99,7 +90,7 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") COMMAND tar cvh . -C ${CMAKE_SOURCE_DIR}/docker/impala_base/ . | ${DOCKER_BUILD} -t impala_base_${build_type} --build-arg BASE_IMAGE=${DISTRO_BASE_IMAGE} - --build-arg INSTALL_OS_PACKAGES_ARGS=${INSTALL_OS_PACKAGES_ARGS} - + --build-arg INSTALL_OS_PACKAGES_ARGS=${install_os_packages_args} - WORKING_DIRECTORY ${IMPALA_BASE_BUILD_CONTEXT_DIR}/${build_type} DEPENDS impala_base_build_context_${build_type} ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh @@ -109,13 +100,18 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") VERBATIM ) endfunction() - add_base_image(release "") - add_base_image(debug "--debug-build") + add_base_image(release "" "") + add_base_image(release_java11 "" "--java 11") + # Debug images include debug tools + add_base_image(debug "--debug-build" "--install-debug-tools") + add_base_image(debug_java11 "--debug-build" "--install-debug-tools --java 11") # Target to build all docker images. Dependencies are added for each docker image # instantiated below. add_custom_target(docker_images) + add_custom_target(docker_java11_images) add_custom_target(docker_debug_images) + add_custom_target(docker_debug_java11_images) add_custom_target(quickstart_docker_images) set(exported_image_names "") @@ -146,13 +142,20 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") function(add_daemon_docker_images daemon_name) set(release_image ${daemon_name}) set(release_target ${daemon_name}_image) + set(release_java11_image ${daemon_name}_java11) + set(release_java11_target ${daemon_name}_java11_image) set(debug_image ${daemon_name}_debug) set(debug_target ${daemon_name}_debug_image) + set(debug_java11_image ${daemon_name}_debug_java11) + set(debug_java11_target ${daemon_name}_debug_java11_image) add_daemon_docker_image(${release_target} ${daemon_name} ${release_image} release) + add_daemon_docker_image(${release_java11_target} ${daemon_name} ${release_java11_image} release_java11) add_daemon_docker_image(${debug_target} ${daemon_name} ${debug_image} debug) + add_daemon_docker_image(${debug_java11_target} ${daemon_name} ${debug_java11_image} debug_java11) ADD_DEPENDENCIES(docker_images ${release_target}) + ADD_DEPENDENCIES(docker_java11_images ${release_java11_target}) ADD_DEPENDENCIES(docker_debug_images ${debug_target}) - set(exported_image_names "${exported_image_names} ${release_image}" PARENT_SCOPE) + ADD_DEPENDENCIES(docker_debug_java11_images ${debug_java11_target}) endfunction() # Stamp out image targets for all of the Impala daemons. @@ -211,7 +214,8 @@ if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED") # to the docker daemon can be very expensive, so we create a build context # with symlinks add_custom_target(impala_utility_build_context_${build_type} - COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args} --utility-context + COMMAND ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py ${build_context_args} + --utility-context --output-dir ${IMPALA_UTILITY_BUILD_CONTEXT_DIR}/${build_type} DEPENDS impala-profile-tool ${CMAKE_SOURCE_DIR}/docker/setup_build_context.py COMMENT "Creating impala utility build context build_type=${build_type}." VERBATIM diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh index 0c97f93a1..0c863a045 100755 --- a/docker/install_os_packages.sh +++ b/docker/install_os_packages.sh @@ -24,12 +24,12 @@ set -euo pipefail INSTALL_DEBUG_TOOLS=false -USE_JAVA11=false +JAVA_VERSION=8 function print_usage { echo "install_os_packages.sh - Helper script to install OS dependencies" echo "[--install-debug-tools] : Also install debug tools like curl, iproute, etc" - echo "[--use-java11] : Use Java 11 rather than the default Java 8." + echo "[--java <version>] : Use specified Java version rather than the default Java 8." } while [ -n "$*" ] @@ -38,8 +38,9 @@ do --install-debug-tools) INSTALL_DEBUG_TOOLS=true ;; - --use-java11) - USE_JAVA11=true + --java) + JAVA_VERSION="${2-}" + shift; ;; --help|*) print_usage @@ -50,7 +51,7 @@ do done echo "INSTALL_DEBUG_TOOLS=${INSTALL_DEBUG_TOOLS}" -echo "USE_JAVA11=${USE_JAVA11}" +echo "JAVA_VERSION=${JAVA_VERSION}" # This can get more detailed if there are specific steps # for specific versions, but at the moment the distribution @@ -67,7 +68,7 @@ else # Ubuntu 16.04 does not have Java 11 in its package repository, # so exit with an error. - if [[ $DISTRIB_RELEASE == 16.04 ]] && $USE_JAVA11 ; then + if [[ $DISTRIB_RELEASE == 16.04 ]] && [[ $JAVA_VERSION == 11 ]] ; then echo "ERROR: Java 11 is not supported on Ubuntu 16.04" exit 1 fi @@ -85,11 +86,6 @@ fi if [[ $DISTRIBUTION == Ubuntu ]]; then export DEBIAN_FRONTEND=noninteractive apt-get update - if $USE_JAVA11 ; then - apt-get install -y openjdk-11-jre-headless - else - apt-get install -y openjdk-8-jre-headless - fi apt-get install -y \ hostname \ krb5-user \ @@ -97,6 +93,7 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then libsasl2-2 \ libsasl2-modules \ libsasl2-modules-gssapi-mit \ + openjdk-${JAVA_VERSION}-jre-headless \ tzdata if $INSTALL_DEBUG_TOOLS ; then echo "Installing extra debug tools" @@ -111,17 +108,14 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then vim fi elif [[ $DISTRIBUTION == Redhat ]]; then - if $USE_JAVA11 ; then - yum install -y --disableplugin=subscription-manager \ - java-11-openjdk-headless - else - yum install -y --disableplugin=subscription-manager \ - java-1.8.0-openjdk-headless + if [[ $JAVA_VERSION == 8 ]]; then + JAVA_VERSION=1.8.0 fi yum install -y --disableplugin=subscription-manager \ cyrus-sasl-gssapi \ cyrus-sasl-plain \ hostname \ + java-${JAVA_VERSION}-openjdk-headless \ krb5-workstation \ openldap-devel \ procps-ng \ diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py index 82f13e0bf..b46d88043 100755 --- a/docker/setup_build_context.py +++ b/docker/setup_build_context.py @@ -33,17 +33,12 @@ parser.add_argument("--debug-build", help="Setup build context for debug build", parser.add_argument("--utility-context", help="Setup utility build context instead of daemon", action="store_true") +parser.add_argument("--output-dir", help="Directory to use for output") args = parser.parse_args() IMPALA_HOME = os.environ["IMPALA_HOME"] -if args.debug_build: - BUILD_TYPE = "debug" -else: - BUILD_TYPE = "release" -if args.utility_context: - OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context_utility", BUILD_TYPE) -else: - OUTPUT_DIR = os.path.join(IMPALA_HOME, "docker/build_context", BUILD_TYPE) +BUILD_TYPE = "debug" if args.debug_build else "release" +OUTPUT_DIR = args.output_dir IMPALA_TOOLCHAIN_PACKAGES_HOME = os.environ["IMPALA_TOOLCHAIN_PACKAGES_HOME"] IMPALA_GCC_VERSION = os.environ["IMPALA_GCC_VERSION"]
