This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new f54177bb0a Docker test container fixes: remove unused files/folders in
image, and use envs instead of env-file
f54177bb0a is described below
commit f54177bb0a549cdf5773c20f3453f7bb0f68b0ad
Author: Mick Semb Wever <m...@apache.org>
AuthorDate: Mon Jan 8 23:29:57 2024 +0100
Docker test container fixes: remove unused files/folders in image, and use
envs instead of env-file
Adds warning and reduces container requested cpus if over docker daemon
limits.
patch by Mick Semb Wever; reviewed by Brandon Williams for INFRA-25202
---
.build/docker/_docker_init_tests.sh | 8 ++++++++
.build/docker/run-tests.sh | 18 ++++++++----------
.build/docker/ubuntu2004_test.docker | 6 ++++--
3 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/.build/docker/_docker_init_tests.sh
b/.build/docker/_docker_init_tests.sh
index 7c90833582..1efbd87526 100755
--- a/.build/docker/_docker_init_tests.sh
+++ b/.build/docker/_docker_init_tests.sh
@@ -36,4 +36,12 @@ if [ -d "${DIST_DIR}/test/logs" ]; then
find "${DIST_DIR}/test/logs" -type f -name "*.log" | xargs xz -qq
fi
popd >/dev/null
+
+# check/clean containerfs (it can leak on host)
+if [ -d /home/cassandra-tmp/.m2/repository ]; then
+ echo "WARN: /home/cassandra-tmp/.m2/repository exists"
+fi
+# these happen when the image hasn't pre-downloaded all the ccm versions used
in tests
+rm -rf /tmp/ccm-*.tar.gz
+
exit ${status}
\ No newline at end of file
diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh
index 88a570b891..44fc82da06 100755
--- a/.build/docker/run-tests.sh
+++ b/.build/docker/run-tests.sh
@@ -146,6 +146,11 @@ case ${target} in
esac
docker_cpus=$(echo "scale=2; ${cores} / ( ${jenkins_executors} )" | bc)
+docker_cpus_limit=$(docker info | grep CPUs | cut -d" " -f3)
+if (( $(echo "${docker_cpus} > ${docker_cpus_limit}" |bc -l) )) ; then
+ echo "WARNING: requested more cpus (${docker_cpus}) than docker cpu limit
(${docker_cpus_limit}), reducing cpus…"
+ docker_cpus=${docker_cpus_limit}
+fi
# hack: long-test does not handle limited CPUs
if [ "${target}" == "long-test" ] ; then
@@ -156,7 +161,7 @@ else
docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g"
fi
-docker_flags="${docker_flags} --env-file build/env.list -d --rm"
+docker_flags="${docker_flags} -d --rm"
# make sure build_dir is good
mkdir -p ${build_dir}/tmp || true
@@ -186,14 +191,7 @@ else
fi
# the docker container's env
-touch build/env.list
-cat > build/env.list <<EOF
-TEST_SCRIPT=${test_script}
-JAVA_VERSION=${java_version}
-PYTHON_VERSION=${python_version}
-cython=${cython}
-ANT_OPTS="${ANT_OPTS}"
-EOF
+docker_envs="--env TEST_SCRIPT=${test_script} --env
JAVA_VERSION=${java_version} --env PYTHON_VERSION=${python_version} --env
cython=${cython} --env ANT_OPTS=\"${ANT_OPTS}\""
split_str="0_0"
if [[ "${split_chunk}" =~ ^[0-9]+/[0-9]+$ ]]; then
@@ -219,7 +217,7 @@ docker_command="source
\${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_versi
\${CASSANDRA_DIR}/.build/docker/_docker_init_tests.sh ${target}
${split_chunk} ; exit \$?"
# start the container, timeout after 4 hours
-docker_id=$(docker run --name ${container_name} ${docker_flags}
${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h)
+docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs}
${docker_mounts} ${docker_volume_opt} ${image_name} sleep 4h)
echo "Running container ${container_name} ${docker_id}"
diff --git a/.build/docker/ubuntu2004_test.docker
b/.build/docker/ubuntu2004_test.docker
index b593cbf610..7bda7074ef 100644
--- a/.build/docker/ubuntu2004_test.docker
+++ b/.build/docker/ubuntu2004_test.docker
@@ -146,13 +146,15 @@ RUN bash -c 'source ~/env3.7/bin/activate && \
RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg
--print-architecture)
# Initialize the CCM git repo as well as this also can fail to clone
+# TODO Move binary:5.0* to above once GA
RUN /bin/bash -c 'source ~/env3.7/bin/activate && \
ccm create -n 1 -v git:trunk test && ccm remove test && \
ccm create -n 1 -v git:cassandra-5.0 test && ccm remove test && \
- ccm create --quiet -n 1 -v binary:5.0-alpha1 test && ccm remove test '
+ ccm create --quiet -n 1 -v binary:5.0-beta1 test && ccm remove test '
# the .git subdirectories to pip installed cassandra-driver breaks
virtualenv-clone, so just remove them
-RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git
+# and other directories we don't need in image
+RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git
/home/cassandra-tmp/.m2 /tmp/ccm-*.tar.gz
# fix permissions, runtime user has different uid/gid
RUN chmod -R og+wx ${BUILD_HOME}/.ccm ${BUILD_HOME}/env* ${BUILD_HOME}/.cache
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
