This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
The following commit(s) were added to refs/heads/trunk by this push:
new a552785 Fixes to test failures introduced by docker `--cpus` limits
a552785 is described below
commit a55278588bad3063fee9d712bf7f6cd351c9d3ac
Author: Mick Semb Wever <[email protected]>
AuthorDate: Tue Jun 1 12:16:12 2021 +0200
Fixes to test failures introduced by docker `--cpus` limits
- don't apply cpu limits to long-test (the View* tests fail)
- don't inner split long-test, along with the burn tests give them extra
resource allocation
- debug docker_cpus evaluation (crashing on ci-caassandra, not yet
reproduced)
- make git cloning quiet
- increase build timeouts from 20 to 90 minutes (limited cpus can slow
down the queue throughput)
- decrease docker prune time period, as no jobs run for more than 12 hours
anymore
patch by Mick Semb Wever; reviewed by Jacek Lewandowski, Berenguer Blasi
---
build-scripts/cassandra-dtest-pytest-docker.sh | 7 ++++---
build-scripts/cassandra-test-docker.sh | 15 +++++++++++----
jenkins-dsl/cassandra_job_dsl_seed.groovy | 10 +++++-----
3 files changed, 20 insertions(+), 12 deletions(-)
diff --git a/build-scripts/cassandra-dtest-pytest-docker.sh
b/build-scripts/cassandra-dtest-pytest-docker.sh
index 338de2a..cf4977c 100755
--- a/build-scripts/cassandra-dtest-pytest-docker.sh
+++ b/build-scripts/cassandra-dtest-pytest-docker.sh
@@ -16,10 +16,10 @@ if [ "$#" -lt 3 ]; then
export PYTHONIOENCODING=utf-8
export PYTHONUNBUFFERED=true
echo "running: git clone --depth 1 --single-branch --branch=$BRANCH
https://github.com/$REPO/cassandra.git";
- git clone --depth 1 --single-branch --branch=$BRANCH
https://github.com/$REPO/cassandra.git
+ git clone --quiet --depth 1 --single-branch --branch=$BRANCH
https://github.com/$REPO/cassandra.git
cd cassandra
echo "running: git clone --depth 1 --single-branch --branch=$DTEST_BRANCH
$DTEST_REPO"
- git clone --depth 1 --single-branch --branch=$DTEST_BRANCH $DTEST_REPO
+ git clone --quiet --depth 1 --single-branch --branch=$DTEST_BRANCH
$DTEST_REPO
echo "cassandra-dtest-pytest.sh (${1} ${2}) cassandra: `git log -1
--pretty=format:'%h %an %ad %s'`" | tee "${1}-$(echo $2 | sed
's/\//-/')-cassandra.head"
echo "cassandra-dtest-pytest.sh (${1} ${2}) cassandra-dtest: `git -C
cassandra-dtest log -1 --pretty=format:'%h %an %ad %s'`" | tee -a "${1}-$(echo
$2 | sed 's/\//-/')-cassandra.head"
echo "cassandra-dtest-pytest.sh (${1} ${2}) cassandra-builds: `git -C
../cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" | tee -a
"${1}-$(echo $2 | sed 's/\//-/')-cassandra.head"
@@ -43,6 +43,7 @@ DTEST_REPO=$3
DTEST_BRANCH=$4
EOF
+ set -x # debug, sometimes ${docker_cpus} is not evaluated
# Jenkins agents run multiple executors per machine. `jenkins_executors=1`
is used for anything non-jenkins.
jenkins_executors=1
if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then
@@ -57,7 +58,7 @@ EOF
[[ "$(docker images -q $DOCKER_IMAGE 2>/dev/null)" != "" ]] || docker pull
-q $DOCKER_IMAGE
echo "cassandra-dtest-pytest-docker.sh: running: git clone --single-branch
--depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh
./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET
$SPLIT_CHUNK"
- ID=$(docker run --cpus=${docker_cpus} -m 15g --memory-swap 15g --env-file
env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch
--depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh
./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET
$SPLIT_CHUNK")
+ ID=$(docker run --cpus=${docker_cpus} -m 15g --memory-swap 15g --env-file
env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --quiet
--single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh
./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET
$SPLIT_CHUNK")
# use docker attach instead of docker wait to get output
docker attach --no-stdin $ID
diff --git a/build-scripts/cassandra-test-docker.sh
b/build-scripts/cassandra-test-docker.sh
index 6774dc1..d60a48d 100755
--- a/build-scripts/cassandra-test-docker.sh
+++ b/build-scripts/cassandra-test-docker.sh
@@ -73,11 +73,11 @@ EOF
command -v nproc >/dev/null 2>&1 && cores=$(nproc --all)
# for relevant test targets calculate how many docker containers we should
split the test list over
case $TARGET in
- # test-burn doesn't have enough tests in it to split beyond 8
- "stress-test" | "fqltool-test" | "microbench" | "test-burn")
+ # test-burn doesn't have enough tests in it to split beyond 8, and burn
and long we want a bit more resources anyway
+ "stress-test" | "fqltool-test" | "microbench" | "test-burn" |
"long-test")
docker_runs=1
;;
- "test"| "test-cdc" | "test-compression" | "long-test" | "jvm-dtest" |
"jvm-dtest-upgrade")
+ "test"| "test-cdc" | "test-compression" | "jvm-dtest" |
"jvm-dtest-upgrade")
mem=1
# linux
command -v free >/dev/null 2>&1 && mem=$(free -b | grep Mem: | awk
'{print $2}')
@@ -99,6 +99,12 @@ EOF
INNER_SPLITS=$(( $(echo $SPLIT_CHUNK | cut -d"/" -f2 ) * $docker_runs ))
INNER_SPLIT_FIRST=$(( ( $(echo $SPLIT_CHUNK | cut -d"/" -f1 ) *
$docker_runs ) - ( $docker_runs - 1 ) ))
docker_cpus=$(echo "scale=2; ${cores} / ( ${jenkins_executors} *
${docker_runs} )" | bc)
+ docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g --env-file
env.list -dt"
+
+ # hack: long-test does not handle limited CPUs
+ if [ "$TARGET" == "long-test" ] ; then
+ docker_flags="-m 5g --memory-swap 5g --env-file env.list -dt"
+ fi
# docker login to avoid rate-limiting apache images. credentials are
expected to already be in place
docker login || true
@@ -113,12 +119,13 @@ EOF
inner_split=$(( $INNER_SPLIT_FIRST + ( $i - 1 ) ))
# start the container
echo "cassandra-test-docker.sh: running: git clone --quiet
--single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; bash
./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET
${inner_split}/${INNER_SPLITS}"
- docker_id=$(docker run --cpus=${docker_cpus} -m 5g --memory-swap 5g
--env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "until git clone
--quiet --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO ; do echo
'git clone failed… trying again… ' ; done ;
./cassandra-builds/build-scripts/cassandra-test-docker.sh ${TARGET}
${inner_split}/${INNER_SPLITS}")
+ docker_id=$(docker run ${docker_flags} $DOCKER_IMAGE dumb-init bash
-ilc "until git clone --quiet --single-branch --depth 1 --branch $BUILDSBRANCH
$BUILDSREPO ; do echo 'git clone failed… trying again… ' ; done ;
./cassandra-builds/build-scripts/cassandra-test-docker.sh ${TARGET}
${inner_split}/${INNER_SPLITS}")
# capture logs and pid for container
docker attach --no-stdin $docker_id >
build/test/logs/docker_attach_${i}.log &
PROCESS_IDS+=( $! )
DOCKER_IDS+=( $docker_id )
+break
done
exit_result=0
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy
b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 89ec118..e46c9f5 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -74,7 +74,7 @@ if(binding.hasVariable("CASSANDRA_DTEST_TEST_TARGETS")) {
def dtestDockerImage = 'apache/cassandra-testing-ubuntu2004-java11'
// expected longest job runtime
-def maxJobHours = 18
+def maxJobHours = 12
if(binding.hasVariable("MAX_JOB_HOURS")) {
maxJobHours = ${MAX_JOB_HOURS}
}
@@ -189,7 +189,7 @@ matrixJob('Cassandra-template-test') {
}
wrappers {
timeout {
- noActivity(1200)
+ noActivity(5400)
}
timestamps()
}
@@ -251,7 +251,7 @@ matrixJob('Cassandra-template-dtest-matrix') {
}
wrappers {
timeout {
- noActivity(1200)
+ noActivity(5400)
}
timestamps()
}
@@ -839,7 +839,7 @@ testTargets.each {
}
wrappers {
timeout {
- noActivity(1200)
+ noActivity(5400)
}
timestamps()
}
@@ -947,7 +947,7 @@ archs.each {
}
wrappers {
timeout {
- noActivity(2400)
+ noActivity(5400)
}
timestamps()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
