This is an automated email from the ASF dual-hosted git repository. mck pushed a commit to branch cassandra-5.0 in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push: new f54177bb0a Docker test container fixes: remove unused files/folders in image, and use envs instead of env-file f54177bb0a is described below commit f54177bb0a549cdf5773c20f3453f7bb0f68b0ad Author: Mick Semb Wever <m...@apache.org> AuthorDate: Mon Jan 8 23:29:57 2024 +0100 Docker test container fixes: remove unused files/folders in image, and use envs instead of env-file Adds warning and reduces container requested cpus if over docker daemon limits. patch by Mick Semb Wever; reviewed by Brandon Williams for INFRA-25202 --- .build/docker/_docker_init_tests.sh | 8 ++++++++ .build/docker/run-tests.sh | 18 ++++++++---------- .build/docker/ubuntu2004_test.docker | 6 ++++-- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.build/docker/_docker_init_tests.sh b/.build/docker/_docker_init_tests.sh index 7c90833582..1efbd87526 100755 --- a/.build/docker/_docker_init_tests.sh +++ b/.build/docker/_docker_init_tests.sh @@ -36,4 +36,12 @@ if [ -d "${DIST_DIR}/test/logs" ]; then find "${DIST_DIR}/test/logs" -type f -name "*.log" | xargs xz -qq fi popd >/dev/null + +# check/clean containerfs (it can leak on host) +if [ -d /home/cassandra-tmp/.m2/repository ]; then + echo "WARN: /home/cassandra-tmp/.m2/repository exists" +fi +# these happen when the image hasn't pre-downloaded all the ccm versions used in tests +rm -rf /tmp/ccm-*.tar.gz + exit ${status} \ No newline at end of file diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh index 88a570b891..44fc82da06 100755 --- a/.build/docker/run-tests.sh +++ b/.build/docker/run-tests.sh @@ -146,6 +146,11 @@ case ${target} in esac docker_cpus=$(echo "scale=2; ${cores} / ( ${jenkins_executors} )" | bc) +docker_cpus_limit=$(docker info | grep CPUs | cut -d" " -f3) +if (( $(echo "${docker_cpus} > ${docker_cpus_limit}" |bc -l) )) ; then + echo "WARNING: requested more cpus (${docker_cpus}) than docker cpu limit (${docker_cpus_limit}), reducing cpus…" + docker_cpus=${docker_cpus_limit} +fi # hack: long-test does not handle limited CPUs if [ "${target}" == "long-test" ] ; then @@ -156,7 +161,7 @@ else docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g" fi -docker_flags="${docker_flags} --env-file build/env.list -d --rm" +docker_flags="${docker_flags} -d --rm" # make sure build_dir is good mkdir -p ${build_dir}/tmp || true @@ -186,14 +191,7 @@ else fi # the docker container's env -touch build/env.list -cat > build/env.list <<EOF -TEST_SCRIPT=${test_script} -JAVA_VERSION=${java_version} -PYTHON_VERSION=${python_version} -cython=${cython} -ANT_OPTS="${ANT_OPTS}" -EOF +docker_envs="--env TEST_SCRIPT=${test_script} --env JAVA_VERSION=${java_version} --env PYTHON_VERSION=${python_version} --env cython=${cython} --env ANT_OPTS=\"${ANT_OPTS}\"" split_str="0_0" if [[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]]; then @@ -219,7 +217,7 @@ docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_versi \${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh ${target} ${split_chunk} ; exit \$?" # start the container, timeout after 4 hours -docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h) +docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h) echo "Running container ${container_name} ${docker_id}" diff --git a/.build/docker/ubuntu2004_test.docker b/.build/docker/ubuntu2004_test.docker index b593cbf610..7bda7074ef 100644 --- a/.build/docker/ubuntu2004_test.docker +++ b/.build/docker/ubuntu2004_test.docker @@ -146,13 +146,15 @@ RUN bash -c 'source ~/env3.7/bin/activate && \ RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture) # Initialize the CCM git repo as well as this also can fail to clone +# TODO Move binary:5.0* to above once GA RUN /bin/bash -c 'source ~/env3.7/bin/activate && \ ccm create -n 1 -v git:trunk test && ccm remove test && \ ccm create -n 1 -v git:cassandra-5.0 test && ccm remove test && \ - ccm create --quiet -n 1 -v binary:5.0-alpha1 test && ccm remove test ' + ccm create --quiet -n 1 -v binary:5.0-beta1 test && ccm remove test ' # the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them -RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git +# and other directories we don't need in image +RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git /home/cassandra-tmp/.m2 /tmp/ccm-*.tar.gz # fix permissions, runtime user has different uid/gid RUN chmod -R og+wx ${BUILD_HOME}/.ccm ${BUILD_HOME}/env* ${BUILD_HOME}/.cache --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org