osmith has submitted this change. (
https://gerrit.osmocom.org/c/osmo-ci/+/33988 )
Change subject: jobs/gerrit-binpkgs: don't hardcode distros
......................................................................
jobs/gerrit-binpkgs: don't hardcode distros
Make it possible to set a list of distributions to check in
gerrit-verifications.yml instead of only having one boolean for testing
two hardcoded rpm and deb distributions.
Change-Id: I59487e3dc2f55057de1b6a322f088fff0d18654c
---
M jobs/gerrit-binpkgs.yml
M jobs/gerrit-verifications.yml
M scripts/jenkins-gerrit/comment_generate.py
3 files changed, 109 insertions(+), 61 deletions(-)
Approvals:
laforge: Looks good to me, approved
pespin: Looks good to me, but someone else must approve
Jenkins Builder: Verified
diff --git a/jobs/gerrit-binpkgs.yml b/jobs/gerrit-binpkgs.yml
index 7a4e9e9..ecfbd52 100644
--- a/jobs/gerrit-binpkgs.yml
+++ b/jobs/gerrit-binpkgs.yml
@@ -5,13 +5,10 @@
- project:
name: gerrit-binpkgs
jobs:
- - 'gerrit-binpkgs-{type}'
- type:
- - 'deb'
- - 'rpm'
+ - 'gerrit-binpkgs'
-- job-template:
- name: 'gerrit-binpkgs-{type}'
+- job:
+ name: 'gerrit-binpkgs'
project-type: freestyle
node: osmocom-gerrit
concurrent: true
@@ -21,7 +18,7 @@
days-to-keep: 30
num-to-keep: 1000
description: |
- {type} package job of CI for patches sent to
+ binary packages job (deb/rpm) of CI for patches sent to
<a href="https://gerrit.osmocom.org";>gerrit</a>
</br></br>
Related issue: <a href="https://osmocom.org/issues/2385";>OS#2385</a>
@@ -44,6 +41,9 @@
- string:
name: PROJECT_NAME
description: project to test, e.g. 'libosmocore' (set by pipeline)
+ - string:
+ name: DISTRO
+ description: distribution to build for, e.g. 'debian:12' (set by
pipeline)
scm:
- git:
@@ -62,14 +62,13 @@
builders:
- shell: |-
- # Skip rpm build if .spec.in file is missing
- if [ "{type}" = "rpm" ] && [ -z "$(find code-from-gerrit -name
'*.spec.in')" ]; then
- set +x
- echo
- echo "No *.spec.in file found, skipping rpm build."
- echo
- exit 0
- fi
+ # Make obvious which distro we build for, for humans and for parsing
+ # by scripts/jenkins-gerrit/comment_generate.py
+ set +x
+ echo
+ echo "Building binary packages for distro: '$DISTRO'"
+ echo
+ set -x
# Clone osmo-ci.git
rm -rf osmo-ci
@@ -80,13 +79,6 @@
osmo-ci
git -C osmo-ci log --oneline
- # Get distro from type
- case "{type}" in
- deb) distro="debian:11" ;;
- rpm) distro="almalinux:8" ;;
- *) echo "unknown package format: '{type}'"; exit 1 ;;
- esac
-
# Move code from gerrit to build_srcpkg.py's git cache
cache_dir=osmo-ci/scripts/obs/_cache
mkdir -p $cache_dir
@@ -106,7 +98,7 @@
# Build binary package
./build_binpkg.py \
- --docker "$distro" \
+ --docker "$DISTRO" \
"$PROJECT_NAME"
wrappers:
- ansicolor:
diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml
index b77b0ca..f2c8bec 100644
--- a/jobs/gerrit-verifications.yml
+++ b/jobs/gerrit-verifications.yml
@@ -92,20 +92,21 @@
# Which jobs to run in the pipeline
pipeline_build: true
pipeline_lint: true
- pipeline_binpkgs: true
+ pipeline_binpkgs: "almalinux:8 debian:11"
pipeline_endianness: true
# in alphabetical order
repos:
- asn1c:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
pipeline_lint: false
- - cellmgr-ng
+ - cellmgr-ng:
+ pipeline_binpkgs: "debian:11"
- docker-playground:
pipeline_build: false
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
pipeline_endianness: false
- libasn1c
@@ -125,7 +126,7 @@
- libsmpp34
- libtelnet:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- libusrp
@@ -143,7 +144,7 @@
-e SMPP="$SMPP" \
-e MGCP="$MGCP" \
{docker_img} {timeout_cmd} /build/contrib/jenkins.sh
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- openggsn:
a1_name: GTP
@@ -190,6 +191,7 @@
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd}
/build/contrib/jenkins.sh'
pipeline_endianness: false
+ pipeline_binpkgs: "debian:11"
- osmo-e1-hardware:
a2_name: JOB_TYPE
@@ -206,11 +208,11 @@
-e JOB_TYPE="$JOB_TYPE" \
"$DOCKER_IMG" \
{timeout_cmd} /build/contrib/jenkins.sh
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-ci:
pipeline_build: false
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
pipeline_endianness: false
- osmo-ggsn:
@@ -229,12 +231,13 @@
- osmo-gsm-manuals:
pipeline_endianness: false
+ pipeline_binpkgs: "debian:11"
- osmo_gsup:
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run_ccache} {docker_img_erlang} {timeout_cmd}
/build/contrib/jenkins.sh'
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-hlr
- osmo-iuh
@@ -290,6 +293,7 @@
repos_url: '{gerrit_url}/python/{repos}'
gerrit_project: 'python/{repos}'
pipeline_endianness: false
+ pipeline_binpkgs: "debian:11"
- osmo-sgsn:
a1_name: IU
@@ -309,7 +313,7 @@
repos_url: '{gerrit_url}/erlang/{repos}'
gerrit_project: 'erlang/{repos}'
cmd: '{docker_run} {docker_img_erlang} {timeout_cmd}
/build/contrib/jenkins.sh'
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
pipeline_endianness: false
- osmo-sip-connector
@@ -336,39 +340,41 @@
"$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh
- osmocom-bb:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-tetra:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- - osmo-sysmon
+ - osmo-sysmon:
+ pipeline_binpkgs: "debian:11"
+
- osmo-remsim
- simtrace2
- osmo-opencm3-projects:
cmd: '{docker_run} {docker_img} {timeout_cmd}
/build/contrib/jenkins.sh'
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-asf4-dfu:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-ccid-firmware:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-e1d
- osmo-cbc
- osmo-e1-recorder:
docker_img: '$USER/debian-bullseye-jenkins' # needs python2
(OS#5950)
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- gapk:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-uecups
- osmo-el2tpd:
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
- osmo-smlc:
docker_img: '$USER/debian-bullseye-jenkins' # needs python2
(OS#5950)
@@ -413,7 +419,7 @@
{timeout_cmd} /build/contrib/jenkins.sh
;;
esac
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
pipeline_endianness: false
- osmo-ttcn3-hacks:
@@ -431,12 +437,12 @@
su build -c 'make clean';
su build -c 'make compile';
"
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
pipeline_endianness: false
- dahdi-tools:
pipeline_lint: false
- pipeline_binpkgs: false
+ pipeline_binpkgs: ""
jobs:
- 'gerrit-{repos}'
@@ -459,21 +465,26 @@
name: REPO_URL
description: Clone URL, to be passed to jobs started in the pipeline
default: '{obj:repos_url}'
- - bool:
+ - string:
name: PIPELINE_BINPKGS
- description: Enable the binpkgs job (build deb/rpm packages)
+ description: |
+ Enable the binpkgs job (build deb/rpm packages) for the given space
+ separated distributions. Empty list disables the binpkgs job. The
+ distributions must have binary packages in the osmocom:master
+ repository.
default: '{obj:pipeline_binpkgs}'
- bool:
name: PIPELINE_ENDIANNESS
description: Run struct_endianness.py from libosmocore.git
default: '{obj:pipeline_endianness}'
dsl: |
- def run_job(display_name, job_name, comment_type=null) {{
+ def run_job(display_name, job_name, comment_type=null, distro=null) {{
try {{
echo "pipeline_${{display_name}}: running job ${{job_name}}"
build job: job_name, parameters: [
string(name: "COMMENT_TYPE", value: comment_type),
+ string(name: "DISTRO", value: distro),
string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"),
string(name: "GERRIT_CHANGE_NUMBER", value:
"${{env.GERRIT_CHANGE_NUMBER}}"),
string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"),
@@ -530,24 +541,42 @@
}}
}}
- stage("deb") {{
+ // Generating binpkgs stages is not possible if we want to run
+ // them in parallel and not have another job that just waits
+ // until they complete.
+
+ stage("binpkgs-0") {{
when {{
- expression {{ params.PIPELINE_BINPKGS }}
+ expression {{ params.PIPELINE_BINPKGS.split().size() > 0 }}
}}
steps {{
script {{
- run_job("deb", "gerrit-binpkgs-deb")
+ distro = params.PIPELINE_BINPKGS.split()[0]
+ run_job(distro, "gerrit-binpkgs", null, distro)
}}
}}
}}
- stage("rpm") {{
+ stage("binpkgs-1") {{
when {{
- expression {{ params.PIPELINE_BINPKGS }}
+ expression {{ params.PIPELINE_BINPKGS.split().size() > 1 }}
}}
steps {{
script {{
- run_job("rpm", "gerrit-binpkgs-rpm")
+ distro = params.PIPELINE_BINPKGS.split()[1]
+ run_job(distro, "gerrit-binpkgs", null, distro)
+ }}
+ }}
+ }}
+
+ stage("binpkgs-2") {{
+ when {{
+ expression {{ params.PIPELINE_BINPKGS.split().size() > 2 }}
+ }}
+ steps {{
+ script {{
+ distro = params.PIPELINE_BINPKGS.split()[2]
+ run_job(distro, "gerrit-binpkgs", null, distro)
}}
}}
}}
diff --git a/scripts/jenkins-gerrit/comment_generate.py
b/scripts/jenkins-gerrit/comment_generate.py
index 6689ee4..e96893c 100755
--- a/scripts/jenkins-gerrit/comment_generate.py
+++ b/scripts/jenkins-gerrit/comment_generate.py
@@ -9,8 +9,9 @@
jenkins_url = "https://jenkins.osmocom.org";
re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_0-9]* #[0-9]*")
-re_result = re.compile("^pipeline_([a-zA-Z-_0-9]*): (SUCCESS|FAILED)$")
+re_result = re.compile("^pipeline_([a-zA-Z-_0-9:]*): (SUCCESS|FAILED)$")
re_job_type = re.compile("JOB_TYPE=([a-zA-Z-_0-9]*),")
+re_distro = re.compile("Building binary packages for distro:
'([a-zA-Z0-9:].*)'")
def parse_args():
@@ -31,16 +32,29 @@
return parser.parse_args()
-def stage_from_job_name(job_name):
+def stage_binpkgs_from_url(job_url):
+ """ Multiple gerrit-binpkgs jobs may be started to build binary packages
+ for multiple distributions. It is not clear from the job name / URL of
+ a job for which distro it is building, so read it from the log output.
+ :returns: a distro like "debian:12" """
+ global re_distro
+
+ url = f"{job_url}/consoleText"
+ with urllib.request.urlopen(url) as response:
+ content = response.read().decode("utf-8")
+ match = re_distro.search(content)
+ assert match, f"couldn't find distro name in log: {url}"
+ return match.group(1)
+
+
+def stage_from_job_name(job_name, job_url):
if job_name == "gerrit-verifications-comment":
# The job that runs this script. Don't include it in the summary.
return None
if job_name == "gerrit-lint":
return "lint"
- if job_name == "gerrit-binpkgs-deb":
- return "deb"
- if job_name == "gerrit-binpkgs-rpm":
- return "rpm"
+ if job_name == "gerrit-binpkgs":
+ return stage_binpkgs_from_url(job_url)
if job_name == "gerrit-pipeline-endianness":
return "endianness"
if job_name.endswith("-build"):
@@ -69,7 +83,7 @@
job_name = match.split(" ")[2]
job_id = int(match.split(" ")[3].replace("#", ""))
job_url = f"{jenkins_url}/jenkins/job/{job_name}/{job_id}"
- stage = stage_from_job_name(job_name)
+ stage = stage_from_job_name(job_name, job_url)
if stage:
ret[stage] = {"url": job_url, "name": job_name, "id":
job_id}
--
To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/33988
To unsubscribe, or for help writing mail filters, visit
https://gerrit.osmocom.org/settings
Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-Change-Id: I59487e3dc2f55057de1b6a322f088fff0d18654c
Gerrit-Change-Number: 33988
Gerrit-PatchSet: 2
Gerrit-Owner: osmith <[email protected]>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: laforge <[email protected]>
Gerrit-Reviewer: osmith <[email protected]>
Gerrit-Reviewer: pespin <[email protected]>
Gerrit-MessageType: merged
