Hashar has uploaded a new change for review.
https://gerrit.wikimedia.org/r/301084
Change subject: (WIP) overhaul / document debian-glue jobs (WIP)
......................................................................
(WIP) overhaul / document debian-glue jobs (WIP)
The 'debian-glue' job has been polished recently and compared to the job
template '{name}-debian-glue' it:
* Uses Gerrit to clone to have the proper branches (T117869)
* Points DIST to whatever distribution is in debian/changelog
This patch overhaul the legacy/broken job template and replace it with
'debian-glue'.
* Add a lot of inline comments.
* Remove '{name}-debian-glue'
* Remove unnused JJB macros
* Have comments wrapped with '***' to match jenkins-debian-glue output
* Zuul 'success-url' points to the build page instead of the console. It
has the artifacts / test reports
* Port all existing per repo job to the generic 'debian-glue' one
* Jobs are non voting. Can be made voting later on?
* Promote the job to test pipeline for mediawiki/debian (T122978)
* Same for Nodepool
Bug: T117869
Bug: T122978
Change-Id: I58721940cf92d119829f36109116045075815998
---
M jjb/operations-debs.yaml
M zuul/layout.yaml
2 files changed, 94 insertions(+), 205 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/integration/config
refs/changes/84/301084/1
diff --git a/jjb/operations-debs.yaml b/jjb/operations-debs.yaml
index 2c3c0b9..2d5bfa9 100644
--- a/jjb/operations-debs.yaml
+++ b/jjb/operations-debs.yaml
@@ -1,34 +1,20 @@
-- builder:
- name: debian-glue-generate-git-snapshot
- builders:
- - shell: /usr/bin/generate-git-snapshot
+# Jobs to build a Debian package using the set of shell scripts from
+# http://jenkins-debian-glue.org/
+#
+# Merely a very useful wrapper / set of conventions around cowbuilder which on
+# CI slaves are setup via the puppet module package_builder.
+#
+# Packages are build against the distribution mentionned in debian/changelog
+# and we recognize the -wikimedia flavor such as 'jessie-wikimedia'.
+#
+# See upstream:
+# http://jenkins-debian-glue.org/docs/
+# https://github.com/mika/jenkins-debian-glue/tree/master/scripts
-- builder:
- name: debian-glue-build-and-provide-package
- builders:
- - shell: |
- export distribution={distribution}
- export REPOSITORY=/data/project/debianrepo
- export REPOS=jenkins-debian-glue
-
- # Use Wikimedia hooks provided by puppet ::package_builder
- # See below for more detailled explanations
- export PBUILDER_CONFIG=/etc/pbuilderrc
- . /etc/pbuilderrc
- export COWBUILDER_BASE="$BASEPATH"
-
- /usr/bin/build-and-provide-package
-
-- builder:
- name: debian-glue-generate-reprepro-codename
- builders:
- - shell: /usr/bin/generate-reprepro-codename
-
-- builder:
- name: debian-glue-increase-version-number
- builders:
- - shell: /usr/bin/increase-version-number
-
+# Lintian checks packages for common errors. Generate a JUnit report that is
+# then consumed by Jenkins Junit publisher plugin.
+#
+# https://lintian.debian.org/
- builder:
name: debian-glue-lintian
builders:
@@ -41,6 +27,10 @@
/usr/bin/lintian-junit-report --filename lintian-source.txt
*.changes | tee lintian-source.xml
set +o pipefail
+# Piuparts test installation / removal of a package
+#
+# https://piuparts.debian.org/
+#
- builder:
name: debian-glue-piuparts
builders:
@@ -51,42 +41,8 @@
piuparts_tap piuparts.txt > piuparts.tap
#
-# JOB TEMPLATES
+# Generic job template around jenkins-debian-glue
#
-
-- job-template:
- name: '{name}-debian-glue'
- #name: '{name}-debian-glue-{distribution}'
- defaults: use-remoteonly-zuul
- node: DebianGlue && DebianJessie
- concurrent: true
- triggers:
- - zuul
-
- # Clone into ./source
- scm:
- - git:
- url: '$ZUUL_URL/$ZUUL_PROJECT'
- branches:
- - '$ZUUL_COMMIT'
- refspec: '$ZUUL_REF'
- basedir: 'source'
-
- builders:
- - debian-glue-generate-git-snapshot
- - debian-glue-build-and-provide-package:
- distribution: '{distribution}'
- - debian-glue-lintian
- - debian-glue-piuparts
-
- publishers: &debian-glue-publishers
- - junit:
- results: 'lintian-*.xml'
- - tap:
- results: piuparts.tap
- - archive:
- artifacts:
'*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,lintian-*.txt,lintian-*.xml,piuparts.txt,piuparts.tap'
-
- job-template:
name: 'debian-glue'
node: DebianGlue && DebianJessie
@@ -96,8 +52,18 @@
builders:
- shell: |
# Clone from Gerrit to $WORKSPACE /source
+ #
+ # Have to first clone from Gerrit to have all the branches
+ # (debian/upstream/master...) to be at their proper tip. The git
+ # repositories on the zuul-merger instances have their head set at
+ # arbitrary merge commits. Zuul cloner then checkout the
+ # appropriate patch.
+ #
+ # The inlined clone map is to force zuul-cloner to use /source as a
+ # workspace, else it would clone under /source/$ZUUL_PROJECT (the
+ # default).
zuul-cloner --version
- echo "Nuking previous build source tree"
+ echo "*** Nuking previous build source tree ***"
rm -fR source
zuul-cloner \
--color \
@@ -108,18 +74,28 @@
https://gerrit.wikimedia.org/r/p \
"$ZUUL_PROJECT"
- shell: |
+ # jenkins-debian-glue expects the job to be run with the Jenkins
+ # git plugin which exposes metadata via GIT_ environement
+ # variables.
export GIT_BRANCH="$ZUUL_BRANCH"
export GIT_COMMIT="$ZUUL_COMMIT"
+ # FIXME would be for postmerge pipeline later on?
if [ "$ZUUL_PIPELINE" == "gate-and-submit" ]; then
export SKIP_DCH=true
fi
/usr/bin/generate-git-snapshot
- shell: |
# Skip repository setup
+ # FIXME: later on we could have postmerge to publish the resulting
+ # package to some kind of central repository by using REPOSITORY
+ # and REPOS.
export BUILD_ONLY=yes
+ # Always build against the debian/changelog indicated distribution,
+ # gives freedom to developers to pick the Distribution the package
+ # will be build against.
export distribution=$(dpkg-parsechangelog --show-field
distribution -lsource/debian/changelog)
- echo "Distribution set from debian/changelog to $distribution"
+ echo "*** Distribution set from debian/changelog to $distribution
***"
# Use Wikimedia hooks provided by puppet ::package_builder
export PBUILDER_CONFIG=/etc/pbuilderrc
@@ -128,124 +104,43 @@
# Wikimedia flavor. We point jenkins-debian-glue cowbuilder path
# to the canonical image, ie have jessie-wikimedia to point to
# base-jessie-amd64.cow and init hook
+ #
# Since $WIKIMEDIA is set, the hook will inject apt.wikimedia.org
+ #
+ # NOTE: build-and-provide-package invokes cowbuilder with sudo
+ # which strips out every environment variables. They have to be
+ # explicitly whitelisted on Wikitech sudo policy at
+ # https://wikitech.wikimedia.org/wiki/Special:NovaSudoer
+ #
+ # Example:
+ # env_keep+=distribution
+ #
export DIST=$distribution
. /etc/pbuilderrc
export COWBUILDER_BASE="$BASEPATH"
export BUILDRESULT="$WORKSPACE/binaries"
/usr/bin/build-and-provide-package
+ # Linters
- debian-glue-lintian
- debian-glue-piuparts
- publishers: *debian-glue-publishers
-
-- project:
- name: 'debian-glue'
- jobs:
- - 'debian-glue'
+ publishers:
+ # FIXME should switch to xunit publisher to get rid of UNSTABLE status
+ # since on Gerrit patchset the previous build is most probably unrelated.
+ - junit:
+ results: 'lintian-*.xml'
+ # FIXME should convert UNSTABLE to FAILED.
+ - tap:
+ results: piuparts.tap
+ - archive:
+ artifacts:
'*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes,lintian-*.txt,lintian-*.xml,piuparts.txt,piuparts.tap'
#
# PROJECTS
#
-
+# FIXME later on that should be autogenerated for each postmerge job.
+#
- project:
- name: gerrit
+ name: debian-glue
jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-apertium'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-apertium-apy'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-cg3'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-hfst'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-lttoolbox'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-archiva'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-gerrit'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-git-fat'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-ganglia'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-jenkins-debian-glue'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-kafka'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-latexml'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: operations-debs-pybal
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-varnish'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-vips'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
-
-- project:
- name: 'operations-debs-wikistats'
- jobs:
- - '{name}-debian-glue':
- distribution: trusty
+ - 'debian-glue'
diff --git a/zuul/layout.yaml b/zuul/layout.yaml
index be5b348..9ac4a8a 100644
--- a/zuul/layout.yaml
+++ b/zuul/layout.yaml
@@ -645,13 +645,12 @@
# Experimental building of Debian packages
# based on jenkins-debian-glue
- - name: ^.*-debian-glue$
+ - name: ^debian-glue$
voting: false
-
- # Work in progress https://gerrit.wikimedia.org/r/110666
- - name: parsoidsvc-debian-glue
- voting: false
- files: 'debian/.*'
+ # Point directly to artifacts
+ success-pattern:
'https://integration.wikimedia.org/ci/job/{job.name}/{build.number}/'
+ # FIXME maybe restrict just to debian/ changes:
+ #files: 'debian/.*'
- name: erblint-HEAD
files:
@@ -816,10 +815,6 @@
branch: ^master$
success-message: 'SUCCESS Change has been deployed on the EQIAD beta
cluster'
failure-message: 'FAILURE Failed deployment on the EQIAD beta cluster :-/
Please contact a member of the beta project to investigate.'
-
- - name: ^debian-glue$
- # Point directly to artifacts
- success-pattern:
'https://integration.wikimedia.org/ci/job/{job.name}/{build.number}/'
- name: parsoidsvc-source-jsduck-publish
branch: ^master$
@@ -1284,7 +1279,6 @@
- name: integration/zuul
experimental:
- - integration-zuul-debian-glue
- debian-glue
- name: maps/tilerator/deploy
@@ -1347,7 +1341,7 @@
- mediawiki-parsertests-hhvm-jessie
- name: mediawiki/debian
- experimental:
+ test:
- debian-glue
- name: mediawiki/event-schemas
@@ -1937,83 +1931,83 @@
- name: operations/debs/archiva
test:
- - operations-debs-archiva-debian-glue
+ - debian-glue
- name: operations/debs/contenttranslation/apertium
test:
- - operations-debs-apertium-debian-glue
+ - debian-glue
- name: operations/debs/contenttranslation/apertium-apy
test:
- - operations-debs-apertium-apy-debian-glue
+ - debian-glue
- name: operations/debs/contenttranslation/cg3
test:
- - operations-debs-cg3-debian-glue
+ - debian-glue
- name: operations/debs/contenttranslation/hfst
test:
- - operations-debs-hfst-debian-glue
+ - debian-glue
- name: operations/debs/contenttranslation/lttoolbox
test:
- - operations-debs-lttoolbox-debian-glue
+ - debian-glue
- name: operations/debs/debdeploy
- experimental:
+ test:
- debian-glue
- name: operations/debs/ganglia
test:
- - operations-debs-ganglia-debian-glue
+ - debian-glue
- name: operations/debs/gerrit
test:
- - operations-debs-gerrit-debian-glue
+ - debian-glue
- name: operations/debs/git-fat
test:
- - operations-debs-git-fat-debian-glue
+ - debian-glue
- name: operations/debs/jenkins-debian-glue
test:
- - operations-debs-jenkins-debian-glue-debian-glue
+ - debian-glue
- name: operations/debs/kafka
test:
- - operations-debs-kafka-debian-glue
+ - debian-glue
- name: operations/debs/latexml
test:
- - operations-debs-latexml-debian-glue
+ - debian-glue
- name: operations/debs/nodepool
- experimental:
+ test:
- debian-glue
- name: operations/debs/pybal
template:
- name: tox-jessie
test:
- - operations-debs-pybal-debian-glue
+ - debian-glue
- name: operations/debs/varnish
test:
- - operations-debs-varnish-debian-glue
+ - debian-glue
- name: operations/debs/vips
test:
- - operations-debs-vips-debian-glue
+ - debian-glue
- name: operations/debs/wikistats
check:
- php53lint
test:
- php53lint
- - operations-debs-wikistats-debian-glue
+ - debian-glue
gate-and-submit:
- php53lint
- - operations-debs-wikistats-debian-glue
+ - debian-glue
- name: operations/dns
check-voter:
@@ -6921,7 +6915,7 @@
- name: mediawiki/services/parsoid/deploy
test:
- - parsoidsvc-debian-glue
+ - debian-glue
- parsoidsvc-deploy-parse-tool-check-trusty
- parsoidsvc-deploy-roundtrip-test-check-trusty
- parsoidsvc-deploy-npm-node-0.10
--
To view, visit https://gerrit.wikimedia.org/r/301084
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I58721940cf92d119829f36109116045075815998
Gerrit-PatchSet: 1
Gerrit-Project: integration/config
Gerrit-Branch: master
Gerrit-Owner: Hashar <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
