This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
The following commit(s) were added to refs/heads/master by this push:
new dfc3adf In Jenkins, print SHAs for repos used in builds, add
cassandra-website build, and SSH artifacts to nightlies.a.o (instead of
archiving)
dfc3adf is described below
commit dfc3adf28768ded5f84a3f23ff926695263fbd5a
Author: Mick Semb Wever <[email protected]>
AuthorDate: Sun Sep 13 12:09:18 2020 +0200
In Jenkins, print SHAs for repos used in builds, add cassandra-website
build, and SSH artifacts to nightlies.a.o (instead of archiving)
patch by Mick Semb Wever; reviewed by Berenguer Blasi for CASSANDRA-16128
---
docker/jenkins/dtest.sh | 3 +
docker/jenkins/jenkinscommand.sh | 5 +-
jenkins-dsl/cassandra_job_dsl_seed.groovy | 197 ++++++++++++++++++++++++++----
jenkins-dsl/cassandra_pipeline.groovy | 3 +
4 files changed, 184 insertions(+), 24 deletions(-)
diff --git a/docker/jenkins/dtest.sh b/docker/jenkins/dtest.sh
index 417a36b..7f15f4a 100644
--- a/docker/jenkins/dtest.sh
+++ b/docker/jenkins/dtest.sh
@@ -9,4 +9,7 @@ git clone --depth 1 --single-branch --branch=$BRANCH
https://github.com/$REPO/ca
cd cassandra
echo git clone --depth 1 --single-branch --branch=$DTEST_BRANCH $DTEST_REPO
git clone --depth 1 --single-branch --branch=$DTEST_BRANCH $DTEST_REPO
+echo "dtest.sh (${1} ${2}) cassandra: `git log -1 --pretty=format:'%h %an %ad
%s'`" | tee "${1}-$(echo $2 | sed 's/\//-/')-cassandra.head"
+echo "dtest.sh (${1} ${2}) cassandra-dtest: `git -C cassandra-dtest log -1
--pretty=format:'%h %an %ad %s'`" | tee -a "${1}-$(echo $2 | sed
's/\//-/')-cassandra.head"
+echo "dtest.sh (${1} ${2}) cassandra-builds: `git -C ../cassandra-builds log
-1 --pretty=format:'%h %an %ad %s'`" | tee -a "${1}-$(echo $2 | sed
's/\//-/')-cassandra.head"
../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh "$@"
diff --git a/docker/jenkins/jenkinscommand.sh b/docker/jenkins/jenkinscommand.sh
index 7ce3c65..1c7f9e1 100644
--- a/docker/jenkins/jenkinscommand.sh
+++ b/docker/jenkins/jenkinscommand.sh
@@ -1,6 +1,6 @@
#!/bin/sh
if [ "$#" -lt 7 ]; then
- echo "Usage: jenkinscommand.sh GITHUB_USER BRANCH DTEST_REPO_URL
DTEST_BRANCH BUILDS_REPO_URL BUILDS_BRANCH DOCKER_IMAGE [target]"
+ echo "Usage: jenkinscommand.sh REPO BRANCH DTEST_REPO_URL DTEST_BRANCH
BUILDS_REPO_URL BUILDS_BRANCH DOCKER_IMAGE [target]"
exit 1
fi
BUILDSREPO=$5
@@ -21,7 +21,6 @@ ID=$(docker run --env-file env.list -dt $DOCKER_IMAGE
dumb-init bash -ilc "git c
# use docker attach instead of docker wait to get output
docker attach --no-stdin $ID
status="$?"
-echo "$ID done (${status}), copying files"
if [ "$status" -ne 0 ] ; then
echo "$ID failed (${status}), debug…"
@@ -36,6 +35,8 @@ if [ "$status" -ne 0 ] ; then
dmesg
else
echo "$ID done (${status}), copying files"
+ # dtest.sh meta
+ docker cp "$ID:/home/cassandra/cassandra/${TARGET}-$(echo $SPLIT_CHUNK |
sed 's/\//-/')-cassandra.head" .
# pytest results
docker cp $ID:/home/cassandra/cassandra/cassandra-dtest/nosetests.xml .
# pytest logs
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy
b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 93a5710..50e0102 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -2,6 +2,8 @@
//
// Common Vars and Branch List
//
+// Help on syntax see https://jenkinsci.github.io/job-dsl-plugin/
+//
// To update the variable via the Jenkins UI, use the EnvInject plugin
// example:
https://github.com/apache/cassandra-builds/pull/19#issuecomment-610822772
//
@@ -122,13 +124,25 @@ matrixJob('Cassandra-template-artifacts') {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b
${buildsBranch} ${buildsRepo}")
+ shell("""
+ git clean -xdff ;
+ git clone --depth 1 --single-branch -b ${buildsBranch}
${buildsRepo} ;
+ echo "cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`" ;
+ """)
}
publishers {
- archiveArtifacts('build/apache-cassandra-*.tar.gz,
build/apache-cassandra-*.jar, build/apache-cassandra-*.pom,
build/cassandra*.deb, build/cassandra*.rpm,
build/**/eclipse_compiler_checks.txt')
- archiveJavadoc {
- javadocDir 'build/javadoc'
- keepAll false
+ publishOverSsh {
+ server('Nightlies') {
+ transferSet {
+ sourceFiles("build/apache-cassandra-*.tar.gz,
build/apache-cassandra-*.jar, build/apache-cassandra-*.pom,
build/cassandra*.deb, build/cassandra*.rpm")
+ remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/")
+ }
+ transferSet {
+ sourceFiles("build/javadoc/**/*")
+ remoteDirectory("cassandra/\${JOB_NAME}/")
+ }
+ }
+ failOnError(false)
}
extendedEmail {
recipientList('[email protected]')
@@ -204,11 +218,16 @@ job('Cassandra-template-test') {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b
${buildsBranch} ${buildsRepo}")
+ shell("""
+ git clean -xdff ;
+ git clone --depth 1 --single-branch -b ${buildsBranch}
${buildsRepo} ;
+ echo "cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`" ;
+ echo "\${JOB_BASE_NAME}: `git log -1 --pretty=format:'%h %an
%ad %s'`" > \${JOB_BASE_NAME}.head ;
+ """)
}
publishers {
archiveArtifacts {
-
pattern('build/test/**/TEST-*.xml,build/**/eclipse_compiler_checks.txt')
+ pattern('build/test/**/TEST-*.xml, **/*.head')
allowEmpty()
fingerprint()
}
@@ -217,9 +236,18 @@ job('Cassandra-template-test') {
publishTestStabilityData()
}
}
+ publishOverSsh {
+ server('Nightlies') {
+ transferSet {
+ sourceFiles("build/test/logs/**")
+ remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/")
+ }
+ }
+ failOnError(false)
+ }
postBuildTask {
task('.', """
- echo "Finding job process orphans…"; if pgrep -af
"${JOB_BASE_NAME}"; then pkill -9 -f "${JOB_BASE_NAME}"; fi;
+ echo "Finding job process orphans…"; if pgrep -af
"\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi;
echo "Cleaning project…"; git clean -xdff ;
echo "Pruning docker…" ; docker system prune -f --filter
"until=${maxJobHours}h" ;
echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs
../../* ;
@@ -266,11 +294,29 @@ matrixJob('Cassandra-template-dtest-matrix') {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b
${buildsBranch} ${buildsRepo} ; git clone --depth 1 --single-branch
${dtestRepo}")
+ shell("""
+ git clean -xdff ;
+ git clone --depth 1 --single-branch -b ${buildsBranch}
${buildsRepo} ;
+ echo "cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`" ;
+ echo "\${JOB_BASE_NAME}: `git log -1 --pretty=format:'%h %an
%ad %s'`" > \${JOB_BASE_NAME}.head ;
+ """)
}
publishers {
+ publishOverSsh {
+ server('Nightlies') {
+ transferSet {
+
sourceFiles("**/nosetests.xml,**/test_stdout.txt,**/ccm_logs.tar.xz")
+ remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/")
+ }
+ transferSet {
+ sourceFiles("build/javadoc/**/*")
+ remoteDirectory("cassandra/\${JOB_NAME}/")
+ }
+ }
+ failOnError(false)
+ }
archiveArtifacts {
- pattern('**/test_stdout.txt,**/nosetests.xml,**/ccm_logs.tar.xz')
+ pattern('**/nosetests.xml, **/*.head')
allowEmpty()
fingerprint()
}
@@ -337,11 +383,14 @@ matrixJob('Cassandra-template-cqlsh-tests') {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch
${dtestRepo}")
+ shell("""
+ git clean -xdff ;
+ echo "\${JOB_BASE_NAME}: `git log -1 --pretty=format:'%h %an
%ad %s'`" > \${JOB_BASE_NAME}.head ;
+ """)
}
publishers {
archiveArtifacts {
- pattern('**/cqlshlib.xml,**/nosetests.xml')
+ pattern('**/cqlshlib.xml,**/nosetests.xml, **/*.head')
allowEmpty()
fingerprint()
}
@@ -352,7 +401,7 @@ matrixJob('Cassandra-template-cqlsh-tests') {
}
postBuildTask {
task('.', """
- echo "Finding job process orphans…"; if pgrep -af
"${JOB_BASE_NAME}"; then pkill -9 -f "${JOB_BASE_NAME}"; fi;
+ echo "Finding job process orphans…"; if pgrep -af
"\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi;
echo "Cleaning project…"; git clean -xdff ;
echo "Pruning docker…" ; docker system prune -f --filter
"until=${maxJobHours}h" ;
echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs
../../* ;
@@ -564,10 +613,27 @@ matrixJob('Cassandra-devbranch-artifacts') {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b
${buildsBranch} ${buildsRepo}")
+ shell("""
+ git clean -xdff ;
+ git clone --depth 1 --single-branch -b ${buildsBranch}
${buildsRepo} ;
+ echo "cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`" ;
+ """)
shell('./cassandra-builds/build-scripts/cassandra-artifacts.sh')
}
publishers {
+ publishOverSsh {
+ server('Nightlies') {
+ transferSet {
+ sourceFiles("build/apache-cassandra-*.tar.gz,
build/apache-cassandra-*.jar, build/apache-cassandra-*.pom,
build/cassandra*.deb, build/cassandra*.rpm")
+ remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/")
+ }
+ transferSet {
+ sourceFiles("build/javadoc/**/*")
+ remoteDirectory("cassandra/\${JOB_NAME}/")
+ }
+ }
+ failOnError(false)
+ }
postBuildTask {
task('.', """
echo "Cleaning project…"; git clean -xdff ;
@@ -627,12 +693,26 @@ testTargets.each {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b
${buildsBranch} ${buildsRepo}")
+ shell("""
+ git clean -xdff ;
+ git clone --depth 1 --single-branch -b ${buildsBranch}
${buildsRepo} ;
+ echo "cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`" ;
+ echo "Cassandra-devbranch-${targetName}: `git log -1
--pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head ;
+ """)
shell("./cassandra-builds/build-scripts/cassandra-test.sh
${targetName}")
}
publishers {
+ publishOverSsh {
+ server('Nightlies') {
+ transferSet {
+ sourceFiles("build/test/logs/**")
+
remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/")
+ }
+ }
+ failOnError(false)
+ }
archiveArtifacts {
-
pattern('build/test/**/TEST-*.xml,build/**/eclipse_compiler_checks.txt')
+ pattern('build/test/**/TEST-*.xml, **/*.head')
allowEmpty()
fingerprint()
}
@@ -641,7 +721,7 @@ testTargets.each {
}
postBuildTask {
task('.', """
- echo "Finding job process orphans…"; if pgrep -af
"${JOB_BASE_NAME}"; then pkill -9 -f "${JOB_BASE_NAME}"; fi;
+ echo "Finding job process orphans…"; if pgrep -af
"\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi;
echo "Cleaning project…"; git clean -xdff ;
echo "Pruning docker…" ; docker system prune -f --filter
"until=${maxJobHours}h" ;
echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs
../../* ;
@@ -719,12 +799,26 @@ dtestTargets.each {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff ; git clone --depth 1 --single-branch -b
${buildsBranch} ${buildsRepo}")
+ shell("""
+ git clean -xdff ;
+ git clone --depth 1 --single-branch -b ${buildsBranch}
${buildsRepo} ;
+ echo "cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`" ;
+ echo "Cassandra-devbranch-${targetName}: `git log -1
--pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-${targetName}.head ;
+ """)
shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh
\$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch}
\$DOCKER_IMAGE ${targetName} \${split}/${splits}")
}
publishers {
+ publishOverSsh {
+ server('Nightlies') {
+ transferSet {
+ sourceFiles("**/test_stdout.txt,**/ccm_logs.tar.xz")
+
remoteDirectory("cassandra/\${JOB_NAME}/\${BUILD_NUMBER}/")
+ }
+ }
+ failOnError(false)
+ }
archiveArtifacts {
-
pattern('**/test_stdout.txt,**/nosetests.xml,**/ccm_logs.tar.xz')
+ pattern('**/nosetests.xml, **/*.head')
allowEmpty()
fingerprint()
}
@@ -794,19 +888,22 @@ matrixJob('Cassandra-devbranch-cqlsh-tests') {
}
steps {
buildDescription('', buildDescStr)
- shell("git clean -xdff")
+ shell("""
+ git clean -xdff ;
+ echo "Cassandra-devbranch-cqlsh-tests: `git log -1
--pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-cqlsh-tests.head ;
+ """)
shell('./pylib/cassandra-cqlsh-tests.sh $WORKSPACE')
}
publishers {
archiveArtifacts {
- pattern('**/cqlshlib.xml,**/nosetests.xml')
+ pattern('**/cqlshlib.xml,**/nosetests.xml, **/*.head')
allowEmpty()
fingerprint()
}
archiveJunit('**/cqlshlib.xml,**/nosetests.xml')
postBuildTask {
task('.', """
- echo "Finding job process orphans…"; if pgrep -af
"${JOB_BASE_NAME}"; then pkill -9 -f "${JOB_BASE_NAME}"; fi;
+ echo "Finding job process orphans…"; if pgrep -af
"\${JOB_BASE_NAME}"; then pkill -9 -f "\${JOB_BASE_NAME}"; fi;
echo "Cleaning project…"; git clean -xdff ;
echo "Pruning docker…" ; docker system prune -f --filter
"until=${maxJobHours}h" ;
echo "Reporting disk usage…"; df -h ; du -hs ../* ; du -hs
../../* ;
@@ -847,3 +944,59 @@ pipelineJob('Cassandra-devbranch') {
}
}
}
+
+////////////////////////////////////////////////////////////
+//
+// Jobs for Other Cassandra Projects
+//
+////////////////////////////////////////////////////////////
+
+job('cassandra-website') {
+ description(jobDescription)
+ label('git-websites')
+ compressBuildLog()
+ logRotator {
+ numToKeep(10)
+ artifactNumToKeep(10)
+ }
+ wrappers {
+ timeout {
+ noActivity(300)
+ }
+ timestamps()
+ }
+ properties {
+ githubProjectUrl('https://github.com/apache/cassandra-website/')
+ priority(1)
+ }
+ scm {
+ git {
+ remote {
+
url('https://gitbox.apache.org/repos/asf/cassandra-website.git')
+ credentials('jenkins (master pub key)')
+ }
+ branch('master')
+ extensions {
+ cleanAfterCheckout()
+ }
+ }
+ }
+ triggers {
+ scm('H/5 * * * *')
+ }
+ steps {
+ buildDescription('', buildDescStr)
+ // the chmod below is a hack for INFRA-20814
+ // for debugging it can be useful to add a `git show --stat HEAD`
before the push
+ shell("""
+ git checkout asf-staging ;
+ git reset --hard origin/master ;
+ docker-compose build --build-arg UID=`id -u` --build-arg
GID=`id -g` cassandra-website ;
+ chmod -R 777 src content ;
+ docker-compose run cassandra-website ;
+ git add content/ src/doc/ ;
+ git commit -a -m "generate docs for `git rev-parse --short
HEAD`" ;
+ git push -f origin asf-staging ;
+ """)
+ }
+}
diff --git a/jenkins-dsl/cassandra_pipeline.groovy
b/jenkins-dsl/cassandra_pipeline.groovy
index 12a7428..dcb23c8 100644
--- a/jenkins-dsl/cassandra_pipeline.groovy
+++ b/jenkins-dsl/cassandra_pipeline.groovy
@@ -243,10 +243,13 @@ pipeline {
echo "sha: ${commit_head_sha}; msg: ${commit_head_msg}"
}
slackSend channel: '#cassandra-builds-patches', message: ":apache:
<${env.BUILD_URL}|${currentBuild.fullDisplayName}> completed:
${currentBuild.result}.
<https://github.com/${REPO}/cassandra/commit/${commit_head_sha}|${REPO}
${commit_head_sha}>\n${commit_head_msg}"
+ sh "echo \"Build based on: `cat *.head`\""
+ sh "echo \"cassandra-builds at: `git -C cassandra-builds log -1
--pretty=format:'%h %an %ad %s'`\""
}
post {
always {
archiveArtifacts artifacts: 'cassandra-test-report.txt',
fingerprint: true
+ sshPublisher(publishers: [sshPublisherDesc(configName:
'Nightlies', transfers: [sshTransfer(remoteDirectory:
'cassandra/${JOB_NAME}/${BUILD_NUMBER}/', sourceFiles:
'cassandra-test-report.txt, TESTS-TestSuites.xml')])])
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
