This is an automated email from the ASF dual-hosted git repository. mck pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
The following commit(s) were added to refs/heads/trunk by this push: new eab310b Add agent_scripts/ for reporting and cleaning agents in a jenkins installation with permament agents eab310b is described below commit eab310bd76329be5d47c7a8c4e8837bbb3e2fff0 Author: Mick Semb Wever <m...@apache.org> AuthorDate: Sat Apr 20 22:10:49 2024 +0200 Add agent_scripts/ for reporting and cleaning agents in a jenkins installation with permament agents This scripts are not embedded into the in-tree Jenkinsfile's so that they can be more easily edited. (They are infrastructure related, rather than release branch related.) Also solves CASSANDRA-18130 patch by Mick Semb Wever; reviewed by Brandon Williams for CASSANDRA-19558 --- jenkins-dsl/agent_scripts/agent_report.sh | 30 +++ jenkins-dsl/agent_scripts/docker_agent_cleaner.sh | 52 +++++ jenkins-dsl/agent_scripts/docker_image_pruner.py | 62 +++++ jenkins-dsl/cassandra_job_dsl_seed.groovy | 264 +++++++++------------- jenkins-dsl/cassandra_pipeline.groovy | 2 +- 5 files changed, 257 insertions(+), 153 deletions(-) diff --git a/jenkins-dsl/agent_scripts/agent_report.sh b/jenkins-dsl/agent_scripts/agent_report.sh new file mode 100644 index 0000000..d190b4f --- /dev/null +++ b/jenkins-dsl/agent_scripts/agent_report.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Report agent hardware and storage +# +# CASSANDRA-18130 + +echo "----" +echo $(date) +echo "${JOB_NAME} ${BUILD_NUMBER} ${STAGE_NAME}" +echo +du -xmh / 2>/dev/null | sort -rh | head -n 30 +echo +df -h +echo +docker system df -v \ No newline at end of file diff --git a/jenkins-dsl/agent_scripts/docker_agent_cleaner.sh b/jenkins-dsl/agent_scripts/docker_agent_cleaner.sh new file mode 100644 index 0000000..1da2259 --- /dev/null +++ b/jenkins-dsl/agent_scripts/docker_agent_cleaner.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Cleans jenkins agents. Primarily used by ci-cassandra.a.o +# +# First argument is `maxJobHours`, all docker objects older than this are pruned +# +# Assumes a CI running multiple C* branches and other jobs + + +# pre-conditions +command -v docker >/dev/null 2>&1 || { error 1 "docker needs to be installed"; } +command -v virtualenv >/dev/null 2>&1 || { error 1 "virtualenv needs to be installed"; } +(docker info >/dev/null 2>&1) || { error 1 "docker needs to running"; } +[ -f "./docker_image_pruner.py" ] || { error 1 "./docker_image_pruner.py must exist"; } + +# arguments +maxJobHours=12 +[ "$#" -gt 0 ] && maxJobHours=$1 + +error() { + echo >&2 $2; + set -x + exit $1 +} + +echo -n "docker system prune --all --force --filter \"until=${maxJobHours}h\" : " +docker system prune --all --force --filter "until=${maxJobHours}h" +if !( pgrep -xa docker &> /dev/null || pgrep -af "build/docker" &> /dev/null || pgrep -af "cassandra-builds/build-scripts" &> /dev/null ) ; then + echo -n "docker system prune --force : " + docker system prune --force || true ; +fi; + +virtualenv -p python3 -q .venv +source .venv/bin/activate +pip -q install requests +python docker_image_pruner.py +deactivate \ No newline at end of file diff --git a/jenkins-dsl/agent_scripts/docker_image_pruner.py b/jenkins-dsl/agent_scripts/docker_image_pruner.py new file mode 100644 index 0000000..9b8060f --- /dev/null +++ b/jenkins-dsl/agent_scripts/docker_image_pruner.py @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Removes all apache/cassandra_ in-tree test images that are not from branch HEADs + +import subprocess +import hashlib +import os +import requests + +# str.removeprefix not available until python3.9 +def remove_prefix(input_string, prefix): + return input_string[len(prefix):] if prefix and input_string.startswith(prefix) else input_string + +def prune_docker_images(): + + docker_images = subprocess.run(['docker', 'images', '--filter', 'reference=apache/cassandra*', '--format', '{{.Repository}}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout.splitlines() + + debug(f"found images: {docker_images}") + md5sums = set() + for branch in ['cassandra-5.0','trunk']: + for docker_image in docker_images: + dockerfile=remove_prefix(docker_image,'apache/cassandra-') + debug(f"checking {branch}/.build/docker/{dockerfile}") + md5sums.add(fetch_url_md5sum(f"https://raw.githubusercontent.com/apache/cassandra/{branch}/.build/docker/{dockerfile}")) + + + if 0 < len(md5sums): + debug(f"in use md5sums are: {md5sums}") + + docker_tags = subprocess.run(['docker', 'images', '--filter', 'reference=apache/cassandra*', '--format', '{{.Repository}}:{{.Tag}}'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout.splitlines() + + debug(f"local images are: {docker_tags}") + for docker_tag in docker_tags: + debug(docker_tag) + if subprocess.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', docker_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).stdout.strip() not in md5sums: + print(f"Pruning {docker_tag}") + subprocess.run(['docker', 'rmi', docker_tag], check=False) + +def fetch_url_md5sum(url): + return hashlib.md5(requests.get(url).content).hexdigest() + +def debug(line): + if 'DEBUG' in os.environ: + print(line) + +if __name__ == "__main__": + prune_docker_images() \ No newline at end of file diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy index 66084ec..dfbd22b 100755 --- a/jenkins-dsl/cassandra_job_dsl_seed.groovy +++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy @@ -491,27 +491,22 @@ legacyCassandraBranches.each { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…"; git clean -qxdff || echo "failed to clean… continuing…"; - echo "Cleaning processes…" ; - if ! ( pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ) ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/${branchName}/${jobNamePrefix}-artifacts/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -586,26 +581,21 @@ legacyCassandraBranches.each { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours + // agent_report.sh does not log to file or archive to nightlies command(""" - echo "Cleaning project…"; git clean -qxdff -e build/test/jmh-result.json || echo "failed to clean… continuing…" ; - echo "Cleaning processes…" ; - if ! ( pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ) ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; du -xm / 2>/dev/null | sort -rn | head -n 30 ; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/${branchName}/${jobNamePrefix}-${targetName}/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) } } } @@ -689,27 +679,22 @@ legacyCassandraBranches.each { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…"; git clean -qxdff || echo "failed to clean… continuing…"; - echo "Cleaning processes…" ; - if ! ( pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ) ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/${branchName}/${jobNamePrefix}-${targetArchName}/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -772,27 +757,22 @@ legacyCassandraBranches.each { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…"; git clean -qxdff || echo "failed to clean… continuing…"; - echo "Cleaning processes…" ; - if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/${branchName}/${jobNamePrefix}-cqlsh-tests/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -883,7 +863,7 @@ matrixJob('Cassandra-devbranch-before-5-artifacts') { } parameters { stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') - stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') + stringParam('BRANCH', 'cassandra-4.1', 'The branch of cassandra to checkout, must be based off before cassandra-5.0') } properties { githubProjectUrl(mainRepo) @@ -923,7 +903,7 @@ matrixJob('Cassandra-devbranch-before-5-artifacts') { server('Nightlies') { transferSet { sourceFiles("console.log.xz,build/apache-cassandra-*.tar.gz, build/apache-cassandra-*.jar, build/apache-cassandra-*.pom, build/cassandra*.deb, build/cassandra*.rpm") - remoteDirectory("cassandra/devbranch/Cassandra-devbranch-artifacts/\${BUILD_NUMBER}/\${JOB_NAME}/") + remoteDirectory("cassandra/devbranch/Cassandra-devbranch-before-5-artifacts/\${BUILD_NUMBER}/\${JOB_NAME}/") } retry(9, 5000) } @@ -934,26 +914,21 @@ matrixJob('Cassandra-devbranch-before-5-artifacts') { markBuildUnstable(false) postBuildStep { stopOnFailure(false) - executeOn('BOTH') + executeOn('AXES') results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…"; git clean -qxdff ; - echo "Cleaning processes…" ; - if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type -f -atime +3 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/devbranch/Cassandra-devbranch-artifacts/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -999,7 +974,7 @@ testTargets.each { } parameters { stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') - stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') + stringParam('BRANCH', 'cassandra-4.1', 'The branch of cassandra to checkout, must be based off before cassandra-5.0') } properties { githubProjectUrl(mainRepo) @@ -1031,7 +1006,7 @@ testTargets.each { echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%H %an %ad %s'`" ; """) shell(""" - echo "Cassandra-devbranch-${targetName}) cassandra: `git log -1 --pretty=format:'%H %an %ad %s'`" > Cassandra-devbranch-${targetName}.head ; + echo "Cassandra-devbranch-before-5-${targetName}) cassandra: `git log -1 --pretty=format:'%H %an %ad %s'`" > Cassandra-devbranch-before-5-${targetName}.head ; ./cassandra-builds/build-scripts/cassandra-test-docker.sh \${REPO} \${BRANCH} ${buildsRepo} ${buildsBranch} ${testDockerImage} ${targetName} \${split}${_testSplits} ; ./cassandra-builds/build-scripts/cassandra-test-report.sh ; xz TESTS-TestSuites.xml ; @@ -1044,7 +1019,7 @@ testTargets.each { server('Nightlies') { transferSet { sourceFiles("TESTS-TestSuites.xml.xz,build/test/logs/**,build/test/jmh-result.json") - remoteDirectory("cassandra/devbranch/Cassandra-devbranch-${targetName}/\${BUILD_NUMBER}/\${JOB_NAME}/") + remoteDirectory("cassandra/devbranch/Cassandra-devbranch-before-5-${targetName}/\${BUILD_NUMBER}/\${JOB_NAME}/") } retry(9, 5000) } @@ -1062,27 +1037,22 @@ testTargets.each { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…"; git clean -qxdff ${targetName == 'microbench' ? '-e build/test/jmh-result.json' : ''}; - echo "Cleaning processes…" ; - if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ;\n\ - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; du -xm / 2>/dev/null | sort -rn | head -n 30 ; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/devbranch/Cassandra-devbranch-${targetName}/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -1121,7 +1091,7 @@ archs.each { } parameters { stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') - stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') + stringParam('BRANCH', 'cassandra-4.1', 'The branch of cassandra to checkout, must be based off before cassandra-5.0') stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') stringParam('DTEST_BRANCH', 'trunk', 'The branch of cassandra-dtest to checkout') stringParam('DOCKER_IMAGE', "${dtestDockerImage}", 'Docker image for running dtests') @@ -1178,7 +1148,7 @@ archs.each { git clean -qxdff ; git clone --depth 1 --single-branch -b ${buildsBranch} ${buildsRepo} ; echo "cassandra-builds at: `git -C cassandra-builds log -1 --pretty=format:'%H %an %ad %s'`" ; - echo "Cassandra-devbranch-${targetArchName}) cassandra: `git log -1 --pretty=format:'%H %an %ad %s'`" > Cassandra-devbranch-${targetArchName}.head ; + echo "Cassandra-devbranch-before-5-${targetArchName}) cassandra: `git log -1 --pretty=format:'%H %an %ad %s'`" > Cassandra-devbranch-before-5-${targetArchName}.head ; """) shell(""" ./cassandra-builds/build-scripts/cassandra-dtest-pytest-docker.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch} \$DOCKER_IMAGE ${targetName} \${split}/${splits} ; @@ -1191,7 +1161,7 @@ archs.each { server('Nightlies') { transferSet { sourceFiles("console.log.xz,**/nosetests.xml,**/test_stdout.txt.xz,**/ccm_logs.tar.xz") - remoteDirectory("cassandra/devbranch/Cassandra-devbranch-${targetArchName}/\${BUILD_NUMBER}/\${JOB_NAME}/") + remoteDirectory("cassandra/devbranch/Cassandra-devbranch-before-5-${targetArchName}/\${BUILD_NUMBER}/\${JOB_NAME}/") } retry(9, 5000) } @@ -1207,27 +1177,22 @@ archs.each { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…" ; git clean -qxdff ; - echo "Cleaning processes…" ; - if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/devbranch/Cassandra-devbranch-${targetArchName}/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -1258,7 +1223,7 @@ matrixJob('Cassandra-devbranch-before-5-cqlsh-tests') { } parameters { stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') - stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') + stringParam('BRANCH', 'cassandra-4.1', 'The branch of cassandra to checkout, must be based off before cassandra-5.0') stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') stringParam('DTEST_BRANCH', 'trunk', 'The branch of cassandra-dtest to checkout') } @@ -1312,7 +1277,7 @@ matrixJob('Cassandra-devbranch-before-5-cqlsh-tests') { server('Nightlies') { transferSet { sourceFiles("console.log.xz,**/test_stdout.txt.xz,**/ccm_logs.tar.xz") - remoteDirectory("cassandra/devbranch/Cassandra-devbranch-cqlsh-tests/\${BUILD_NUMBER}/\${JOB_NAME}/") + remoteDirectory("cassandra/devbranch/Cassandra-devbranch-before-5-cqlsh-tests/\${BUILD_NUMBER}/\${JOB_NAME}/") } retry(9, 5000) } @@ -1328,27 +1293,22 @@ matrixJob('Cassandra-devbranch-before-5-cqlsh-tests') { buildSteps { markBuildUnstable(false) postBuildStep { - executeOn('BOTH') + executeOn('AXES') stopOnFailure(false) results(['SUCCESS','UNSTABLE','FAILURE','NOT_BUILT','ABORTED']) buildSteps { - shell { - // docker needs to (soon or later) prune its volumes too, but that can only be done when the agent is idle - // if the agent is busy, just prune everything that is older than maxJobHours - command(""" - echo "Cleaning project…"; git clean -qxdff ; - echo "Cleaning processes…" ; - if ! ( pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ) ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; - echo "Pruning docker…" ; - docker volume ls -qf dangling=true | xargs -r docker rm -v || true ; - if pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts" ; then docker system prune --force --filter "until=${maxJobHours}h" || true ; else docker system prune --all --force --volumes || true ; fi; - echo "Reporting disk usage…"; df -h ; - echo "Cleaning tmp…"; - find . -type d -name tmp -delete 2>/dev/null ; - find /tmp -type f -atime +2 -user jenkins -and -not -exec fuser -s {} ';' -and -delete 2>/dev/null || echo clean tmp failed ; - echo "For test report and logs see https://nightlies.apache.org/cassandra/devbranch/Cassandra-devbranch-cqlsh-tests/\${BUILD_NUMBER}/\${JOB_NAME}/" - """) - } + shell { + // agent_report.sh does not log to file or archive to nightlies + command(""" + echo "Cleaning processes…" + if ! (pgrep -xa docker || pgrep -af "cassandra-builds/build-scripts") ; then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ; + echo "Pruning docker for '${JOB_NAME}' on ${NODE_NAME}…" + cd cassandra-builds/jenkins-dsl/agent_scripts/ + bash docker_agent_cleaner.sh ${maxJobHours} + bash agent_report.sh + git clean -qxdff -e build/test/jmh-result.json || true + """) + } } } } @@ -1371,7 +1331,7 @@ pipelineJob('Cassandra-devbranch-before-5') { } parameters { stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') - stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') + stringParam('BRANCH', 'cassandra-4.1', 'The branch of cassandra to checkout, must be based off before cassandra-5.0') stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') stringParam('DTEST_BRANCH', 'trunk', 'The branch of cassandra-dtest to checkout') stringParam('DOCKER_IMAGE', "${dtestDockerImage}", 'Docker image for running dtests') diff --git a/jenkins-dsl/cassandra_pipeline.groovy b/jenkins-dsl/cassandra_pipeline.groovy index 86da48f..7013d87 100644 --- a/jenkins-dsl/cassandra_pipeline.groovy +++ b/jenkins-dsl/cassandra_pipeline.groovy @@ -1,4 +1,4 @@ -// Lgeacy: only for branches older than 5.0 +// Legacy: only for branches older than 5.0 // // Before 5.0 Cassandra-devbranch needs custom Jenkinsfile because of the parameters passed into the build jobs. // --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org