This is an automated email from the ASF dual-hosted git repository. mck pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
commit 4ca24701e105226ca35da1adc7ff70b56a36fce2 Author: Mick Semb Wever <[email protected]> AuthorDate: Sun Apr 11 12:34:41 2021 +0200 Split dockerised test runs into parallel docker container runs Improvements - build arm64 docker images on-the-fly (they are not deployed anywhere yet) - only pull the docker images if they are not found locally (saves time!) - return internal exit codes - configure apt to retry on network failures - avoid running empty test splits - remove per-agent throttle (as they are now dockerised) patch by Mick Semb Wever; reviewed by Tomek Łasica for CASSANDRA-16604 --- build-scripts/cassandra-dtest-pytest-docker.sh | 2 + build-scripts/cassandra-test-docker.sh | 132 ++++++++++++++++++------- build-scripts/cassandra-test.sh | 62 ++++++++---- docker/buster-image.docker | 3 + docker/centos8-image.docker | 2 +- docker/jessie-image.docker | 3 + docker/testing/ubuntu1910_j11.docker | 3 + docker/testing/ubuntu2004_j11.docker | 3 + jenkins-dsl/cassandra_job_dsl_seed.groovy | 34 ++++--- 9 files changed, 178 insertions(+), 66 deletions(-) diff --git a/build-scripts/cassandra-dtest-pytest-docker.sh b/build-scripts/cassandra-dtest-pytest-docker.sh old mode 100644 new mode 100755 index 1e2efe6..c1ed7a0 --- a/build-scripts/cassandra-dtest-pytest-docker.sh +++ b/build-scripts/cassandra-dtest-pytest-docker.sh @@ -45,6 +45,7 @@ EOF # docker login to avoid rate-limiting apache images. credentials are expected to already be in place docker login || true + [[ "$(docker images -q $DOCKER_IMAGE 2>/dev/null)" != "" ]] || docker pull -q $DOCKER_IMAGE echo "cassandra-dtest-pytest-docker.sh: running: git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET $SPLIT_CHUNK" ID=$(docker run -m 15g --memory-swap 15g --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh $TARGET $SPLIT_CHUNK") @@ -76,4 +77,5 @@ EOF fi docker rm $ID + exit $status fi \ No newline at end of file diff --git a/build-scripts/cassandra-test-docker.sh b/build-scripts/cassandra-test-docker.sh old mode 100644 new mode 100755 index c2e0973..4d2cd6a --- a/build-scripts/cassandra-test-docker.sh +++ b/build-scripts/cassandra-test-docker.sh @@ -1,7 +1,7 @@ #!/bin/bash # # A wrapper script to cassandra-test.sh -# that runs it in docker, collecting results. +# that split the test list into multiple docker runs, collecting results. # # The docker image used is normally based from those found in docker/testing/ # @@ -22,14 +22,19 @@ if [ "$#" -lt 3 ]; then fi java -version javac -version - echo "running: git clone --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git"; - git clone --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git + echo "running: git clone --quiet --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git"; + until git clone --quiet --depth 1 --single-branch --branch=$BRANCH https://github.com/$REPO/cassandra.git ; do echo "git clone failed… trying again… " ; done cd cassandra echo "cassandra-test.sh (${1} ${2}) cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" | tee "${1}-$(echo $2 | sed 's/\//-/')-cassandra.head" echo "cassandra-test.sh (${1} ${2}) cassandra-builds: `git -C ../cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" | tee -a "${1}-$(echo $2 | sed 's/\//-/')-cassandra.head" - bash ../cassandra-builds/build-scripts/cassandra-test.sh "$@" + ../cassandra-builds/build-scripts/cassandra-test.sh "$@" if [ -d build/test/logs ]; then find build/test/logs -type f -name "*.log" | xargs xz -qq ; fi else + + # pre-conditions + command -v docker >/dev/null 2>&1 || { echo >&2 "docker needs to be installed"; exit 1; } + (docker info >/dev/null 2>&1) || { echo >&2 "docker needs to running"; exit 1; } + # start the docker container if [ "$#" -lt 5 ]; then echo "Usage: cassandra-test-docker.sh REPO BRANCH BUILDS_REPO_URL BUILDS_BRANCH DOCKER_IMAGE [target] [split_chunk]" @@ -38,8 +43,8 @@ else BUILDSREPO=$3 BUILDSBRANCH=$4 DOCKER_IMAGE=$5 - TARGET=$6 - SPLIT_CHUNK=$7 + TARGET=${6:-"test"} + SPLIT_CHUNK=${7:-"1/1"} # Setup JDK java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}') @@ -59,37 +64,98 @@ BRANCH=$2 JAVA_VERSION=${java_version} EOF + # for relevant test targets calculate how many docker containers we should split the test list over + case $TARGET in + # test-burn doesn't have enough tests in it to split beyond 8 + "stress-test" | "fqltool-test" | "microbench" | "test-burn") + docker_runs=1 + ;; + "test"| "test-cdc" | "test-compression" | "long-test" | "jvm-dtest" | "jvm-dtest-upgrade") + cores=1 + cores=$(nproc --all) + mem=1 + # linux + command -v free >/dev/null 2>&1 && mem=$(free -b | grep Mem: | awk '{print $2}') + # macos + sysctl -n hw.memsize >/dev/null 2>&1 && mem=$(sysctl -n hw.memsize) + # Jenkins agents run multiple executors per machine. `jenkins_executors=1` is used for anything non-jenkins. + jenkins_executors=1 + if [[ ! -z ${JENKINS_URL+x} ]] && [[ ! -z ${NODE_NAME+x} ]] ; then + jenkins_executors=$(curl -s "${JENKINS_URL}/computer/${NODE_NAME}/api/json?pretty=true" | grep 'numExecutors' | awk -F' : ' '{print $2}' | cut -d',' -f1) + fi + max_docker_runs_by_cores=$( echo "sqrt( $cores / $jenkins_executors )" | bc ) + max_docker_runs_by_mem=$(( $mem / ( 5 * 1024 * 1024 * 1024 * $jenkins_executors ) )) + docker_runs=$(( $max_docker_runs_by_cores < $max_docker_runs_by_mem ? $max_docker_runs_by_cores : $max_docker_runs_by_mem )) + docker_runs=$(( $docker_runs < 1 ? 1 : $docker_runs )) + ;; + *) + echo "unregconized \"$target\"" + exit 1 + ;; + esac + + # Break up the requested split chunk into a number of concurrent docker runs, as calculated above + # This will typically be between one to four splits. Five splits would require >25 cores and >25GB ram + INNER_SPLITS=$(( $(echo $SPLIT_CHUNK | cut -d"/" -f2 ) * $docker_runs )) + INNER_SPLIT_FIRST=$(( ( $(echo $SPLIT_CHUNK | cut -d"/" -f1 ) * $docker_runs ) - ( $docker_runs - 1 ) )) + # docker login to avoid rate-limiting apache images. credentials are expected to already be in place docker login || true + [[ "$(docker images -q $DOCKER_IMAGE 2>/dev/null)" != "" ]] || docker pull -q $DOCKER_IMAGE - echo "cassandra-test-docker.sh: running: git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET $SPLIT_CHUNK" - ID=$(docker run -m 15g --memory-swap 15g --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET $SPLIT_CHUNK") + mkdir -p build/test/logs + declare -a DOCKER_IDS + declare -a PROCESS_IDS + declare -a STATUSES - # use docker attach instead of docker wait to get output - docker attach --no-stdin $ID - status="$?" + for i in `seq 1 $docker_runs` ; do + inner_split=$(( $INNER_SPLIT_FIRST + ( $i - 1 ) )) + # start the container + echo "cassandra-test-docker.sh: running: git clone --quiet --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO; bash ./cassandra-builds/build-scripts/cassandra-test-docker.sh $TARGET ${inner_split}/${INNER_SPLITS}" + docker_id=$(docker run -m 5g --memory-swap 5g --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "until git clone --quiet --single-branch --depth 1 --branch $BUILDSBRANCH $BUILDSREPO ; do echo 'git clone failed… trying again… ' ; done ; ./cassandra-builds/build-scripts/cassandra-test-docker.sh ${TARGET} ${inner_split}/${INNER_SPLITS}") - if [ "$status" -ne 0 ] ; then - echo "$ID failed (${status}), debug…" - docker inspect $ID - echo "–––" - docker logs $ID - echo "–––" - docker ps -a - echo "–––" - docker info - echo "–––" - dmesg - else - echo "$ID done (${status}), copying files" - # test meta - docker cp "$ID:/home/cassandra/cassandra/${TARGET}-$(echo $SPLIT_CHUNK | sed 's/\//-/')-cassandra.head" . - # test results - mkdir -p build/test - docker cp $ID:/home/cassandra/cassandra/build/test/output/. build/test/output - # test logs - docker cp $ID:/home/cassandra/cassandra/build/test/logs/. build/test/logs - fi + # capture logs and pid for container + docker attach --no-stdin $docker_id > build/test/logs/docker_attach_${i}.log & + PROCESS_IDS+=( $! ) + DOCKER_IDS+=( $docker_id ) + done + + exit_result=0 + i=0 + for process_id in "${PROCESS_IDS[@]}" ; do + # wait for each container to complete + docker_id=${DOCKER_IDS[$i]} + inner_split=$(( $INNER_SPLIT_FIRST + $i )) + cat build/test/logs/docker_attach_$(( $i + 1 )).log + tail -F build/test/logs/docker_attach_$(( $i + 1 )).log & + tail_process=$! + wait $process_id + status=$? + PROCESS_IDS+=( $status ) + kill $tail_process + + if [ "$status" -ne 0 ] ; then + echo "${docker_id} failed (${status}), debug…" + docker inspect ${docker_id} + echo "–––" + docker logs ${docker_id} + echo "–––" + docker ps -a + echo "–––" + docker info + echo "–––" + dmesg + exit_result=1 + else + echo "${docker_id} done (${status}), copying files" + docker cp "$docker_id:/home/cassandra/cassandra/${TARGET}-${inner_split}-${INNER_SPLITS}-cassandra.head" . + docker cp $docker_id:/home/cassandra/cassandra/build/test/output/. build/test/output + docker cp $docker_id:/home/cassandra/cassandra/build/test/logs/. build/test/logs + fi + docker rm $docker_id + ((i++)) + done - docker rm $ID + xz build/test/logs/docker_attach_*.log + exit $exit_result fi diff --git a/build-scripts/cassandra-test.sh b/build-scripts/cassandra-test.sh index ae84f4c..0bb428d 100755 --- a/build-scripts/cassandra-test.sh +++ b/build-scripts/cassandra-test.sh @@ -5,6 +5,11 @@ set -o errexit set -o pipefail set -o nounset +# pre-conditions +command -v ant >/dev/null 2>&1 || { echo >&2 "ant needs to be installed"; exit 1; } +command -v git >/dev/null 2>&1 || { echo >&2 "git needs to be installed"; exit 1; } +[ -f "build.xml" ] || { echo >&2 "build.xml must exist"; exit 1; } + # lists all tests for the specific test type _list_tests() { local -r classlistprefix="$1" @@ -26,7 +31,7 @@ _timeout_for() { _build_all_dtest_jars() { cd $TMP_DIR - git clone --depth 1 --no-single-branch https://gitbox.apache.org/repos/asf/cassandra.git cassandra-dtest-jars + until git clone --quiet --depth 1 --no-single-branch https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done cd cassandra-dtest-jars for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 cassandra-4.0 trunk; do git checkout $branch @@ -36,10 +41,24 @@ _build_all_dtest_jars() { done cd ../.. rm -fR ${TMP_DIR}/cassandra-dtest-jars - ant dtest-jar + ant clean dtest-jar ls -l build/dtest*.jar } +_run_testlist() { + local _target_prefix=$1 + local _testlist_target=$2 + local _split_chunk=$3 + local _test_timeout=$4 + testlist="$( _list_tests "${_target_prefix}" | _split_tests "${_split_chunk}")" + if ! [[ -z "$testlist" ]]; then + ant clean jar + ant $_testlist_target -Dtest.classlistprefix="${_target_prefix}" -Dtest.classlistfile=<(echo "${testlist}") -Dtest.timeout="${_test_timeout}" -Dtmp.dir="${TMP_DIR}" || echo "failed ${_target_prefix} ${$_testlist_target}" + else + echo Skipping ${_target_prefix} ${_testlist_target}, no tests in split ${_split_chunk} + fi +} + _main() { # parameters local -r target="${1:-}" @@ -74,50 +93,53 @@ _main() { export TMP_DIR="$(pwd)/tmp" mkdir -p ${TMP_DIR} - ant clean jar case $target in "stress-test") - # hard fail on test compilation, put dont fail the test run as unstable test reports are processed - ant stress-build-test + # hard fail on test compilation, but dont fail the test run as unstable test reports are processed + ant clean stress-build-test jar ant $target -Dtmp.dir="$(pwd)/tmp" || echo "failed $target" ;; "fqltool-test") - # hard fail on test compilation, put dont fail the test run so unstable test reports are processed - ant fqltool-build-test + # hard fail on test compilation, but dont fail the test run so unstable test reports are processed + ant clean fqltool-build-test jar ant $target -Dtmp.dir="$(pwd)/tmp" || echo "failed $target" ;; "microbench") - ant $target -Dtmp.dir="$(pwd)/tmp" -Dmaven.test.failure.ignore=true + ant clean $target -Dtmp.dir="$(pwd)/tmp" -Dmaven.test.failure.ignore=true ;; "test") - testlist="$( _list_tests "unit" | _split_tests "${split_chunk}")" - ant testclasslist -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + _run_testlist "unit" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.timeout')" ;; "test-cdc") - testlist=$( _list_tests "unit" | _split_tests "${split_chunk}") - ant testclasslist-cdc -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + _run_testlist "unit" "testclasslist-cdc" "${split_chunk}" "$(_timeout_for 'test.timeout')" ;; "test-compression") - testlist=$( _list_tests "unit" | _split_tests "${split_chunk}") - ant testclasslist-compression -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + _run_testlist "unit" "testclasslist-compression" "${split_chunk}" "$(_timeout_for 'test.timeout')" ;; "test-burn") - testlist=$( _list_tests "burn" | _split_tests "${split_chunk}") - ant testclasslist -Dtest.classlistprefix=burn -Dtest.timeout=$(_timeout_for "test.burn.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + _run_testlist "burn" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.burn.timeout')" ;; "long-test") - testlist=$( _list_tests "long" | _split_tests "${split_chunk}") - ant testclasslist -Dtest.classlistprefix=long -Dtest.timeout=$(_timeout_for "test.long.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + _run_testlist "long" "testclasslist" "${split_chunk}" "$(_timeout_for 'test.long.timeout')" ;; "jvm-dtest") + ant clean jar testlist=$( _list_tests "distributed" | grep -v "upgrade" | _split_tests "${split_chunk}") - ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target" + if ! [[ -z "$testlist" ]]; then + ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + else + echo "Skipping $target, no tests in split ${split_chunk}" + fi ;; "jvm-dtest-upgrade") _build_all_dtest_jars testlist=$( _list_tests "distributed" | grep "upgrade" | _split_tests "${split_chunk}") - ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" -Dtest.runners=1 || echo "failed $target" + if ! [[ -z "$testlist" ]]; then + ant testclasslist -Dtest.classlistprefix=distributed -Dtest.timeout=$(_timeout_for "test.distributed.timeout") -Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo "failed $target" + else + echo "Skipping $target, no tests in split ${split_chunk}" + fi ;; *) echo "unregconised \"$target\"" diff --git a/docker/buster-image.docker b/docker/buster-image.docker index 5139a09..4f554cd 100644 --- a/docker/buster-image.docker +++ b/docker/buster-image.docker @@ -8,6 +8,9 @@ LABEL org.cassandra.buildenv=buster VOLUME ${DEB_DIST_DIR} +# configure apt to retry downloads +RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries + # install deps RUN apt-get update && apt-get -y install \ ant \ diff --git a/docker/centos8-image.docker b/docker/centos8-image.docker index 58f6565..0c7e0c3 100644 --- a/docker/centos8-image.docker +++ b/docker/centos8-image.docker @@ -20,7 +20,7 @@ RUN yum -y install \ sudo \ python3-pip -RUN curl http://mirror.centos.org/centos/7/os/x86_64/Packages/ant-junit-1.9.4-2.el7.noarch.rpm -o ant-junit-1.9.4-2.el7.noarch.rpm +RUN curl -S -s --retry 9 http://mirror.centos.org/centos/7/os/x86_64/Packages/ant-junit-1.9.4-2.el7.noarch.rpm -o ant-junit-1.9.4-2.el7.noarch.rpm RUN rpm -i --nodeps ant-junit-1.9.4-2.el7.noarch.rpm diff --git a/docker/jessie-image.docker b/docker/jessie-image.docker index f3a8b69..d68e632 100644 --- a/docker/jessie-image.docker +++ b/docker/jessie-image.docker @@ -8,6 +8,9 @@ LABEL org.cassandra.buildenv=jessie VOLUME ${DEB_DIST_DIR} +# configure apt to retry downloads +RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries + # install deps RUN apt-get update && apt-get -y install \ ant \ diff --git a/docker/testing/ubuntu1910_j11.docker b/docker/testing/ubuntu1910_j11.docker index 8226d9f..0449852 100644 --- a/docker/testing/ubuntu1910_j11.docker +++ b/docker/testing/ubuntu1910_j11.docker @@ -13,6 +13,9 @@ FROM ubuntu:19.10 MAINTAINER Apache Cassandra <[email protected]> +# configure apt to retry downloads +RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries + # install our python dependencies and some other stuff we need # libev4 libev-dev are for the python driver / libssl-dev is for python3.6 diff --git a/docker/testing/ubuntu2004_j11.docker b/docker/testing/ubuntu2004_j11.docker index d442d21..48bc812 100644 --- a/docker/testing/ubuntu2004_j11.docker +++ b/docker/testing/ubuntu2004_j11.docker @@ -13,6 +13,9 @@ FROM ubuntu:20.04 MAINTAINER Apache Cassandra <[email protected]> +# configure apt to retry downloads +RUN echo 'APT::Acquire::Retries "9";' > /etc/apt/apt.conf.d/80-retries + # install our python dependencies and some other stuff we need # libev4 libev-dev are for the python driver / libssl-dev is for python3.6 diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy index 255f390..444bf59 100644 --- a/jenkins-dsl/cassandra_job_dsl_seed.groovy +++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy @@ -186,9 +186,6 @@ matrixJob('Cassandra-template-test') { } timestamps() } - throttleConcurrentBuilds { - categories(['Cassandra']) - } properties { githubProjectUrl(githubRepo) priorityJobProperty { @@ -456,8 +453,16 @@ cassandraBranches.each { node / scm / branches / 'hudson.plugins.git.BranchSpec' / name(branchName) } steps { + if (arch == "-arm64") { + shell(""" + # docker image has to be built on arm64 (they are not currently published to dockerhub) + cd cassandra-builds/docker/testing ; + docker build -t ${dtestDockerImage}:latest -f ubuntu2004_j11.docker . ; + docker build -t ${testDockerImage}:latest -f ubuntu2004_j11_w_dependencies.docker . + """) + } shell(""" - sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh apache ${branchName} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ; + ./cassandra-builds/build-scripts/cassandra-test-docker.sh apache ${branchName} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ; ./cassandra-builds/build-scripts/cassandra-test-report.sh ; xz TESTS-TestSuites.xml """) @@ -555,7 +560,7 @@ cassandraBranches.each { """) } shell(""" - sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git trunk ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits} ; + ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git trunk ${buildsRepo} ${buildsBranch} ${dtestDockerImage} ${targetName} \${split}/${splits} ; """) } publishers { @@ -821,9 +826,6 @@ testTargets.each { } timestamps() } - throttleConcurrentBuilds { - categories(['Cassandra']) - } parameters { stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') @@ -859,10 +861,18 @@ testTargets.each { git clean -xdff ${targetName == 'microbench' ? '-e build/test/jmh-result.json' : ''}; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%h %an %ad %s'`" ; - echo "Cassandra-devbranch-${targetName}) cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head - """) + echo "Cassandra-devbranch-${targetName}) cassandra: `git log -1 --pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head + """) + if (arch == "-arm64") { + shell(""" + # docker image has to be built on arm64 (they are not currently published to dockerhub) + cd cassandra-builds/docker/testing ; + docker build -t ${dtestDockerImage}:latest -f ubuntu2004_j11.docker . ; + docker build -t ${testDockerImage}:latest -f ubuntu2004_j11_w_dependencies.docker . + """) + } shell(""" - sh ./cassandra-builds/build-scripts/cassandra-test-docker.sh \${REPO} \${BRANCH} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ; + ./cassandra-builds/build-scripts/cassandra-test-docker.sh \${REPO} \${BRANCH} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}/${testSplits} ; ./cassandra-builds/build-scripts/cassandra-test-report.sh ; xz TESTS-TestSuites.xml """) @@ -1001,7 +1011,7 @@ archs.each { """) } shell(""" - sh ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${splits} ; + ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${splits} ; """) } publishers { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
