BIGTOP-1501. come up with a way to use Jenkins plugins/Groovy console to create jobs
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/258c97c6 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/258c97c6 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/258c97c6 Branch: refs/heads/master Commit: 258c97c69e9a326da9e40ebb398d675f0919bc3c Parents: f506701 Author: Roman Shaposhnik <[email protected]> Authored: Sun Dec 14 22:02:33 2014 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Tue Feb 10 13:56:05 2015 -0800 ---------------------------------------------------------------------- bigtop-ci/jenkins/README | 31 ++++ bigtop-ci/jenkins/jobsCreator.groovy | 233 ++++++++++++++++++++++++++++++ 2 files changed, 264 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/258c97c6/bigtop-ci/jenkins/README ---------------------------------------------------------------------- diff --git a/bigtop-ci/jenkins/README b/bigtop-ci/jenkins/README new file mode 100644 index 0000000..963f814 --- /dev/null +++ b/bigtop-ci/jenkins/README @@ -0,0 +1,31 @@ +Bigtop's approach to running CI off of Jenkins is simple: we use +Groovy DSL Job plugin extensively so that Jenkins configuration +can be as stateless as possible. All we require from a Jenkins +admin is that the following plugins are installed: + * Amazon EC2 Plugin + * Credentials Plugin + * Disk Usage Plugin + * DynamicAxis Plugin + * Git Plugin + * Gradle plugin + * Hudson Extended Read Permission Plugin + * Job DSL Plugin + * Mailer + * Matrix Authorization Strategy Plugin + * Matrix Project Plugin + * Matrix Reloaded Plugin + * Parameterized Trigger Plugin + * SSH Credentials Plugin + * SSH Slaves plugin + * Timestamper + +Once you have all the above plugins installed, all you have to do in +order to start Bigtop CI is to create one Bootstrap job. Pick a type +of 'Build a free-style software project' when asked by Jenkins and +hook it up to Bigtop's Git repo. Then select 'Process Job DSLs' as +one and only build step, pointing it at + bigtop-ci/jenkins/jobsCreator.groovy +as a DSL script that needs to be looked up in the filesystem. + +That is it! You only need to run it now in order to have all +the required Jenkins jobs. http://git-wip-us.apache.org/repos/asf/bigtop/blob/258c97c6/bigtop-ci/jenkins/jobsCreator.groovy ---------------------------------------------------------------------- diff --git a/bigtop-ci/jenkins/jobsCreator.groovy b/bigtop-ci/jenkins/jobsCreator.groovy new file mode 100644 index 0000000..36c0f3e --- /dev/null +++ b/bigtop-ci/jenkins/jobsCreator.groovy @@ -0,0 +1,233 @@ +// FIXME: it would be nice to extract the following from bigtop.mk on the fly +def bigtopComponents = ["bigtop-groovy", "bigtop-jsvc", "bigtop-tomcat", "bigtop-utils", + "zookeeper", "hadoop", "hbase", "hive", "pig", "crunch", "datafu", + "flume", "giraph", "gridgain-hadoop", "mahout", "oozie", "phoenix", + "solr", "spark", "sqoop", "tachyon", "whirr"] +// FIXME: it would be nice to extract the following from some static configuration file +def targetOS = ["fedora-20", "opensuse-12.3", "ubuntu-14.04"] +def gitUrl = "https://git-wip-us.apache.org/repos/asf/bigtop.git"; +def gitBranch = "master" +def dockerLabel = "docker" +def jobPrefix="Bigtop" +def pkgTestsuites =["TestPackagesBasicsWithRM", "TestPackagesPseudoDistributedServices", + "TestPackagesPseudoDistributedDependency", "TestPackagesPseudoDistributedFileContents", + "TestPackagesPseudoDistributedWithRM", "TestPackagesBasics"] + +job { + name "${jobPrefix}-${gitBranch}-All" + description "Top level job that kicks off everything for a complete Bigtop CI run on branch ${gitBranch}" + logRotator(7 /*days to keep */, 10 /* # of builds */, 7 /*days to keep */, 10 /* # of builds */) + label('master') + triggers { + cron("0 3 * * *") + } + + steps { + downstreamParameterized { + trigger("${jobPrefix}-${gitBranch}-" + bigtopComponents.join("-pkg,${jobPrefix}-${gitBranch}-") + "-pkg", + 'UNSTABLE_OR_BETTER', true, ["buildStepFailure": "FAILURE", "failure": "FAILURE", "unstable": "UNSTABLE"]) { + currentBuild() + } + + trigger("${jobPrefix}-${gitBranch}-Repository", + 'UNSTABLE_OR_BETTER', true, ["buildStepFailure": "FAILURE", "failure": "FAILURE", "unstable": "UNSTABLE"]) { + currentBuild() + } + + trigger("${jobPrefix}-${gitBranch}-Packagetest", + 'UNSTABLE_OR_BETTER', true, ["buildStepFailure": "FAILURE", "failure": "FAILURE", "unstable": "UNSTABLE"]) { + currentBuild() + } + } + } +} + +job(type: Matrix) { + name "${jobPrefix}-${gitBranch}-Repository" + description "Top level job that creates final repository from packages built off of ${gitBranch} branch for all the matrix" + logRotator(2 /*days to keep */, 2 /* # of builds */, 2 /*days to keep */, 2 /* # of builds */) + label('master') + + steps { + shell(''' +#!/bin/bash -ex +env +rm -rf * /var/tmp/* || : + +# By default Jenkins uses an internal (unresolvable) EC2 DNS name +JOB_URL="${JOB_URL/#*:/http://bigtop01.cloudera.org:}"; + +export PROJECTS="''' + bigtopComponents.join(' ') + '''" + +mkdir -p packages +pushd packages + for project in $PROJECTS; do + mkdir -p ${project} + pushd ${project} + wget "http://bigtop01.cloudera.org:8080/job/${jobPrefix}-${gitBranch}-${project}-pkg/TARGET_OS=${TARGET_OS},slaves=docker/lastSuccessfulBuild/artifact/*zip*/archive.zip"; + unzip archive.zip + rm archive.zip + popd + done +popd + +if [ -n "find packages -iname '*.src.rpm'`" ] ; then + mkdir -p repo/{RPMS,SRPMS} + mv `find packages -iname "*.src.rpm"` repo/SRPMS/ + mv `find packages -iname "*.rpm"` repo/RPMS/ + + pushd repo + createrepo . + popd + cat > repo/bigtop.repo << __EOT__ +[bigtop] +name=Bigtop +enabled=1 +gpgcheck=0 +type=NONE +baseurl=${JOB_URL}/lastSuccessfulBuild/artifact/repo/ +__EOT__ + + +else + mkdir -p repo/conf + cat > repo/conf/distributions <<__EOT__ +Origin: Bigtop +Label: Bigtop +Suite: stable +Codename: bigtop +Version: 0.3 +Architectures: i386 amd64 source +Components: contrib +Description: Bigtop +__EOT__ + for i in `find packages -name \\*.changes` ; do + reprepro -Vb repo include bigtop $i + done + echo "deb ${JOB_URL}/lastSuccessfulBuild/artifact/repo/ bigtop contrib" > repo/bigtop.list +fi + ''') + } + + publishers { + archiveArtifacts('repo/**/*') + } + + axes { + text('TARGET_OS', targetOS) + label('slaves', dockerLabel) + } +} + +job(type: Matrix) { + name "${jobPrefix}-${gitBranch}-Packagetest" + description "Runs smoke tests on all packages built off of ${gitBranch} branch for all the matrix" + logRotator(4 /*days to keep */, 4 /* # of builds */, 4 /*days to keep */, 4 /* # of builds */) + label('master') + + parameters { + choiceParam("PKG_SUITE", pkgTestsuites) + } + + scm { + git { node -> // is hudson.plugins.git.GitSCM + // node / gitConfigName('Bigtop') + // node / gitConfigEmail('[email protected]') + + remote { + name(gitBranch) + url(gitUrl) + } + branch(gitBranch) + } + } + + steps { + shell(''' +#!/bin/bash + +# Working around SuSE madness +rm -f /etc/zypp/repos.d/*Cloud* || : + +export HADOOP_HOME=/usr/lib/hadoop +export HADOOP_CONF_DIR=/etc/hadoop/conf +export REPO_FILE_URL="http://bigtop01.cloudera.org:8080/view/Bigtop-trunk/job/Bigtop-trunk-Repository/label=${label/-slave/}/lastSuccessfulBuild/artifact/repo/bigtop"; +export REPO_KEY_URL="http://archive.apache.org/dist/incubator/bigtop/bigtop-0.3.0-incubating/repos/GPG-KEY-bigtop"; + +if [ "$label" = "precise-slave" -o "$label" = "quetzal-slave" -o "$label" = "trusty-slave" ]; then + REPO_FILE_URL="${REPO_FILE_URL}.list" +else + REPO_FILE_URL="${REPO_FILE_URL}.repo" +fi + +docker run -u `id -u` -e HOME=/var/lib/jenkins -e BIGTOP_BUILD_STAMP=.${BUILD_NUMBER} \\ + -v `pwd`/build/home:/var/lib/jenkins \\ + -v `pwd`:/ws bigtop/slaves:$TARGET_OS \\ + bash -c '. /etc/profile.d/bigtop.sh; cd /ws ; mvn \\ + -f bigtop-tests/test-execution/package/pom.xml \\ + clean verify \\ + -Dbigtop.repo.file.url="'${REPO_FILE_URL}'" \\ + -Dorg.apache.bigtop.itest.log4j.level=TRACE \\ + -Dlog4j.debug=true \\ + -Dorg.apache.maven-failsafe-plugin.testInclude="**/'${PKG_SUITE}'.*" + ''') + } + + publishers { + archiveJunit('**/bigtop-tests/test-execution/package/target/failsafe-reports/*.xml') { + retainLongStdout() + } + } + + axes { + text('TARGET_OS', targetOS) + label('slaves', dockerLabel) + } +} + +bigtopComponents.each { comp-> + job(type: Matrix) { + println comp + name "${jobPrefix}-${gitBranch}-${comp}-pkg" + description "Builds packages on every platform of the matrix according to the ${gitBranch} branch" + logRotator(7 /*days to keep */, 10 /* # of builds */, 7 /*days to keep */, 10 /* # of builds */) + label('master') + + scm { + git { node -> // is hudson.plugins.git.GitSCM + // node / gitConfigName('Bigtop') + // node / gitConfigEmail('[email protected]') + + remote { + name(gitBranch) + url(gitUrl) + } + branch(gitBranch) + } + } + +// export JAVA_OPTS="-Xmx1536m -Xms256m -XX:MaxPermSize=256m" +// export MAVEN_OPTS="-Xmx1536m -Xms256m -XX:MaxPermSize=256m" +// +// if [ -e /etc/SuSE-release ] ; then +// export LDFLAGS="-lrt" +// fi + steps { + shell(''' + mkdir -p build/home || : + docker run -u `id -u` -e HOME=/var/lib/jenkins -e BIGTOP_BUILD_STAMP=.${BUILD_NUMBER} \\ + -v `pwd`/build/home:/var/lib/jenkins \\ + -v `pwd`:/ws bigtop/slaves:$TARGET_OS \\ + bash -c '. /etc/profile.d/bigtop.sh; cd /ws ; gradle ''' + "'\"${comp}-pkg\"") + } + + publishers { + archiveArtifacts('output/**/*/*') + } + + axes { + text('TARGET_OS', targetOS) + label('slaves', dockerLabel) + } + } +}
