osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ci/+/32016 )
Change subject: jobs/osmocom-release-manuals: add new job ...................................................................... jobs/osmocom-release-manuals: add new job Add a jenkins job that automatically builds manuals for release tags, if they don't exist on the server already. Closes: OS#5902 Related: https://downloads.osmocom.org/docs/ Change-Id: I0ecb238660553c3c857e1b310873eca8a8d09dab --- M .gitignore A jobs/osmocom-release-manuals.yml A scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch A scripts/manuals/publish-manuals-for-tags.sh 4 files changed, 513 insertions(+), 0 deletions(-) Approvals: laforge: Looks good to me, but someone else must approve pespin: Looks good to me, but someone else must approve osmith: Looks good to me, approved Jenkins Builder: Verified diff --git a/.gitignore b/.gitignore index fa28062..3fcbe85 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ _release_tarballs/ _docker_playground _repo_install_test_data/ +_temp_manuals/ scripts/repo-install-test/run-inside-env.sh diff --git a/jobs/osmocom-release-manuals.yml b/jobs/osmocom-release-manuals.yml new file mode 100644 index 0000000..19a8765 --- /dev/null +++ b/jobs/osmocom-release-manuals.yml @@ -0,0 +1,40 @@ +--- +- project: + name: Osmocom-release-manuals + jobs: + - Osmocom-release-manuals + +- job: + name: 'Osmocom-release-manuals' + project-type: freestyle + defaults: global + description: | + Build manuals for all new Osmocom releases + (<a href="https://osmocom.org/issues/5902";>OS#5902</a>). + parameters: + - string: + name: BRANCH_CI + description: | + osmo-ci.git branch + default: 'master' + builders: + - shell: | + scripts/manuals/publish-manuals-for-tags.sh + scm: + - git: + branches: + - '$BRANCH_CI' + url: https://gerrit.osmocom.org/osmo-ci + git-config-name: 'Jenkins Builder' + git-config-email: '[email protected]' + wipe-workspace: true + triggers: + - timed: "@midnight" + publishers: + - email: + notify-every-unstable-build: true + recipients: '[email protected]' + wrappers: + - ssh-agent-credentials: + users: + - docs_ftp.osmocom.org diff --git a/scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch b/scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch new file mode 100644 index 0000000..65a4e89 --- /dev/null +++ b/scripts/manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch @@ -0,0 +1,48 @@ +From ed812f2bd79e7e01c833d90ee279ae4750f82f2f Mon Sep 17 00:00:00 2001 +From: Oliver Smith <[email protected]> +Date: Wed, 8 Mar 2023 17:48:17 +0100 +Subject: [PATCH] build/custom-dblatex.sty: remove DRAFT + +This patch will be used when generating manuals for the tagged +releases, so they don't have the draft watermark. + +Related: OS#5902 +Change-Id: Id9a27b01412b0e615638f20fa5e36f5b114a15d8 +--- + build/Makefile.asciidoc.inc | 4 ---- + build/custom-dblatex.sty | 4 ---- + 2 files changed, 8 deletions(-) + +diff --git a/build/Makefile.asciidoc.inc b/build/Makefile.asciidoc.inc +index 1b7375e..9a9e829 100644 +--- a/build/Makefile.asciidoc.inc ++++ b/build/Makefile.asciidoc.inc +@@ -69,10 +69,6 @@ $(ASCIIDOC_PDF): %.pdf: %.adoc %-docinfo.xml \ + + # TEXINPUTS: find LaTeX includes like \includegraphics{./common/images/sysmocom.pdf} + # in $(OSMO_GSM_MANUALS_DIR). +- @test -n "$(BUILD_RELEASE)" && echo -e "\n\n\ +- NOTE: TO REMOVE DRAFT MODE, YOU NEED TO EDIT build/custom-dblatex.sty\n\ +- and remove three lines starting with '% \"DRAFT\" on first page'\n" \ +- || true + TEXINPUTS="$(OSMO_GSM_MANUALS_DIR)" \ + a2x -vv $(A2X_OPTS) $(notdir $<) + +diff --git a/build/custom-dblatex.sty b/build/custom-dblatex.sty +index 16931ad..0d53bb1 100644 +--- a/build/custom-dblatex.sty ++++ b/build/custom-dblatex.sty +@@ -20,10 +20,6 @@ + \usepackage{alltt} + \usepackage{upquote} + +-% "DRAFT" on first page +-\definecolor{LtGrey}{rgb}{0.875,0.875,0.875} +-\AddToShipoutPicture*{ \AtTextCenter{ \makebox(0,0)[c]{\resizebox{\textwidth}{!}{ \rotatebox{45}{\textsf{\textbf{\color{LtGrey}DRAFT}}}}} } } +- + \def\Company{sysmocom - s.f.m.c. GmbH} + + \def\DBKcover{ +-- +2.34.1 + diff --git a/scripts/manuals/publish-manuals-for-tags.sh b/scripts/manuals/publish-manuals-for-tags.sh new file mode 100755 index 0000000..9d453b8 --- /dev/null +++ b/scripts/manuals/publish-manuals-for-tags.sh @@ -0,0 +1,410 @@ +#!/bin/sh -e +. "$(dirname "$0")/../common.sh" +OSMO_CI_DIR="$(realpath $(dirname "$0")/../..)" +TEMP="$OSMO_CI_DIR/_temp_manuals" +WEB_PATH="/downloads/home/docs/web-files" +SSH_COMMAND="ssh -o UserKnownHostsFile=$TEMP/src/osmo-gsm-manuals/build/known_hosts -p 48" +DOCKER_IMAGE="$USER/debian-bullseye-jenkins" +LOG_PREFIX="::" + +# Releases that were made before shared osmo-gsm-manuals, or where build fails +# for other reasons. +TAGS_IGNORE=" + openbsc:0.9.0, + openbsc:0.9.1, + openbsc:0.9.2, + openbsc:0.9.3, + openbsc:0.9.4, + openbsc:0.9.5, + openbsc:0.9.6, + openbsc:0.9.8, + openbsc:0.9.9, + openbsc:0.9.10, + openbsc:0.9.11, + openbsc:0.9.12, + openbsc:0.9.13, + openbsc:0.9.14, + openbsc:0.9.15, + openbsc:0.9.16, + openbsc:0.10.0, + openbsc:0.10.1, + openbsc:0.11.0, + openbsc:0.12.0, + openbsc:0.13.0, + openbsc:0.14.0, + openbsc:0.15.0, + openbsc:1.0.0, + openbsc:1.1.0, + + osmo-bsc:1.0.1, + osmo-bsc:1.1.0, + osmo-bsc:1.1.1, + osmo-bsc:1.1.2, + osmo-bsc:1.2.0, + osmo-bsc:1.2.1, + osmo-bsc:1.2.2, + osmo-bsc:1.3.0, + osmo-bsc:1.4.0, + osmo-bsc:1.4.1, + osmo-bsc:1.5.0, + osmo-bsc:1.6.0, + + osmo-bts:0.0.1, + osmo-bts:0.1.0, + osmo-bts:0.2.0, + osmo-bts:0.3.0, + osmo-bts:0.4.0, + osmo-bts:0.6.0, + osmo-bts:0.7.0, + osmo-bts:0.8.0, + osmo-bts:0.8.1, + osmo-bts:1.0.0, + osmo-bts:1.0.1, + osmo-bts:1.1.0, + osmo-bts:1.2.0, + + osmo-e1d:0.0.1, + osmo-e1d:0.1.0, + osmo-e1d:0.1.1, + osmo-e1d:0.2.0, + osmo-e1d:0.2.1, + osmo-e1d:0.2.2, + osmo-e1d:0.3.0, + osmo-e1d:0.4.0, + + osmo-ggsn:1.0.0, + osmo-ggsn:1.1.0, + osmo-ggsn:1.2.0, + osmo-ggsn:1.2.1, + osmo-ggsn:1.2.2, + + osmo-gsm-tester:0.1, + + osmo-hlr:0.0.1, + osmo-hlr:0.1.0, + osmo-hlr:0.2.0, + osmo-hlr:0.2.1, + + osmo-mgw:1.0.1, + osmo-mgw:1.0.2, + osmo-mgw:1.1.0, + osmo-mgw:1.2.0, + osmo-mgw:1.2.1, + osmo-mgw:1.3.0, + osmo-mgw:1.4.0, + + osmo-msc:1.0.1, + osmo-msc:1.1.0, + osmo-msc:1.1.1, + osmo-msc:1.1.2, + osmo-msc:1.2.0, + osmo-msc:1.3.0, + osmo-msc:1.3.1, + osmo-msc:1.4.0, + osmo-msc:1.5.0, + osmo-msc:1.6.0, + osmo-msc:1.6.1, + osmo-msc:1.6.2, + osmo-msc:1.6.3, + + osmo-pcap:0.0.1, + osmo-pcap:0.0.2, + osmo-pcap:0.0.3, + osmo-pcap:0.0.4, + osmo-pcap:0.0.5, + osmo-pcap:0.0.6, + osmo-pcap:0.0.7, + osmo-pcap:0.0.8, + osmo-pcap:0.0.9, + osmo-pcap:0.0.10, + osmo-pcap:0.0.11, + osmo-pcap:0.1.0, + osmo-pcap:0.1.1, + osmo-pcap:0.1.2, + osmo-pcap:0.1.3, + + osmo-pcu:0.1, + osmo-pcu:0.1.0, + osmo-pcu:0.2, + osmo-pcu:0.2.0, + osmo-pcu:0.4.0, + osmo-pcu:0.5.0, + osmo-pcu:0.5.1, + + osmo-remsim:0.0, + osmo-remsim:0.1.0, + osmo-remsim:0.2.0, + osmo-remsim:0.2.1, + + osmo-sgsn:0.10.0, + osmo-sgsn:0.10.1, + osmo-sgsn:0.11.0, + osmo-sgsn:0.12.0, + osmo-sgsn:0.13.0, + osmo-sgsn:0.14.0, + osmo-sgsn:0.15.0, + osmo-sgsn:0.9.0, + osmo-sgsn:0.9.1, + osmo-sgsn:0.9.10, + osmo-sgsn:0.9.11, + osmo-sgsn:0.9.12, + osmo-sgsn:0.9.13, + osmo-sgsn:0.9.13+deb1, + osmo-sgsn:0.9.14, + osmo-sgsn:0.9.14-onwaves1, + osmo-sgsn:0.9.15, + osmo-sgsn:0.9.16, + osmo-sgsn:0.9.2, + osmo-sgsn:0.9.3, + osmo-sgsn:0.9.4, + osmo-sgsn:0.9.5, + osmo-sgsn:0.9.6, + osmo-sgsn:0.9.8, + osmo-sgsn:0.9.9, + osmo-sgsn:1.0.1, + osmo-sgsn:1.1.0, + osmo-sgsn:1.10.0, + osmo-sgsn:1.2.0, + osmo-sgsn:1.3.0, + osmo-sgsn:1.4.0, + osmo-sgsn:1.4.1, + osmo-sgsn:1.5.0, + osmo-sgsn:1.6.0, + osmo-sgsn:1.6.1, + + osmo-sip-connector:0.0.1, + osmo-sip-connector:1.1.0, + osmo-sip-connector:1.1.1, + + libosmo-sccp:0.0.1, + libosmo-sccp:0.0.2, + libosmo-sccp:0.0.3, + libosmo-sccp:0.0.4, + libosmo-sccp:0.0.5, + libosmo-sccp:0.0.5.1, + libosmo-sccp:0.0.6, + libosmo-sccp:0.0.6.1, + libosmo-sccp:0.0.6.2, + libosmo-sccp:0.0.6.3, + libosmo-sccp:0.10.0, + libosmo-sccp:0.7.0, + libosmo-sccp:0.8.0, + libosmo-sccp:0.8.1, + libosmo-sccp:0.9.0, + libosmo-sccp:1.0.0, + libosmo-sccp:1.1.0, + libosmo-sccp:1.2.0, + + osmo-trx:0.2.0, + osmo-trx:0.3.0, + osmo-trx:0.4.0, + + pysim:1.0, +" + +mkdir -p \ + "$TEMP" \ + "$TEMP/src" + +check_ssh_auth_sock() { + if [ -z "$SSH_AUTH_SOCK" ]; then + echo "ERROR: SSH_AUTH_SOCK is not set" + exit 1 + fi +} + +# $1: docs dir +get_repo_name_from_docs_dir() { + case "$1" in + osmo-stp) + echo "libosmo-sccp" + ;; + *) + echo "$1" + ;; + esac +} + +# $1: repo name +get_docs_dir_from_repo_name() { + case "$1" in + libosmo-sccp) + echo "osmo-stp" + ;; + *) + echo "$1" + ;; + esac +} + + +# $1: path on server, e.g. "/docs/osmo-bsc" +get_server_ls() { + local dir="$1" + local out="$TEMP/ls$(echo "$dir" | tr / _)" + + echo "$LOG_PREFIX Listing files on server: $dir" + + if [ -e "$out" ]; then + echo "Skipped, file exists: $out" + return + fi + + dir="$(echo "$dir" | sed "s.^/docs.$WEB_PATH.")" + $SSH_COMMAND [email protected] "ls -1 $dir" >"$out" +} + +# $1: repository +get_git_tags() { + local repo="$1" + local out="$TEMP/git_tags_$repo" + + echo "$LOG_PREFIX Getting git tags" + + if [ -e "$out" ]; then + echo "Skipped, file exists: $out" + return + fi + + osmo_git_last_tags "$repo" "all" >"$out" +} + +# $1: docs dir +# $2: tag +manuals_exist() { + local docs_dir="$1" + local tag="$2" + + grep -q "^$tag$" "$TEMP"/ls_docs_"$docs_dir" +} + +# $1: repository +# $2: tag +is_tag_ignored() { + local repo="$1" + local tag="$2" + + case "$TAGS_IGNORE" in + *"$repo:$tag,"*) + return 0 + ;; + esac + + return 1 +} + +# $1: repository +# $2: tag +clone_repo() { + local repo="$1" + local tag="$2" + local gitdir="$TEMP/src/$repo" + + if ! [ -d "$gitdir" ]; then + local url="$(osmo_git_clone_url "$repo")" + echo "$LOG_PREFIX Cloning $url" + git -C "$TEMP/src" clone "$url" "$repo" + fi + + echo "$LOG_PREFIX Checkout $tag" + cd "$gitdir" + git reset --hard HEAD + git checkout "$tag" + git submodule update --init + git clean -dxf + + # Fix depends on packages that don't exist anymore + sed -i 's/dh-systemd \(.*\),//g' debian/control + sed -i 's/python-minimal,//g' debian/control +} + +# $1: repository +# $2: tag +build_publish_manuals() { + local repo="$1" + local tag="$2" + echo "$LOG_PREFIX Building manuals" + + if ! docker run \ + --rm \ + -e "BUILD_RELEASE=1" \ + -e "DEBIAN_FRONTEND=noninteractive" \ + -e "OSMO_GSM_MANUALS_DIR=/opt/osmo-gsm-manuals" \ + -e "OSMO_REPOSITORY=$(get_docs_dir_from_repo_name "$repo")" \ + -e "PUBLISH_REF=$tag" \ + -e "SSH_AUTH_SOCK=/ssh-agent" \ + -v "$OSMO_CI_DIR/scripts/manuals:/manuals" \ + -v "$TEMP/src/$repo/:/build" \ + -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \ + "$DOCKER_IMAGE" \ + sh -ex -c " + apt-get update + + # Install dependencies + case $repo in + *) + apt-get -y build-dep /build + ;; + esac + + # Remove DRAFT in osmo-gsm-manuals + cd /opt/osmo-gsm-manuals/ + patch -p1 < /manuals/0001-build-custom-dblatex.sty-remove-DRAFT.patch + + # Build manuals + cd /build + case $repo in + openbsc) + for dir in manuals/*/; do + su build -c \"make -C \$dir\" + done + ;; + *) + su build -c \"autoreconf -fi\" + su build -c \"./configure --enable-manuals\" + su build -c \"make -j$(nproc)\" + ;; + esac + + + # Publish manuals + case $repo in + openbsc) + for dir in manuals/*/; do + su build -c \"make -C \$dir publish\" + done + ;; + *) + su build -c \"make -C doc/manuals publish\" + ;; + esac + "; then + echo "$LOG_PREFIX Building manuals failed!" + exit 1 + fi +} + +check_ssh_auth_sock +get_server_ls "/docs" + +for docs_dir in $(cat "$TEMP"/ls_docs); do + repo="$(get_repo_name_from_docs_dir "$docs_dir")" + LOG_PREFIX=":: ($repo)" + get_server_ls "/docs/$docs_dir" + get_git_tags "$repo" + + echo "$LOG_PREFIX Building missing manuals" + for tag in $(cat "$TEMP"/git_tags_"$repo"); do + LOG_PREFIX=":: ($repo, $tag)" + if manuals_exist "$docs_dir" "$tag"; then + echo "$LOG_PREFIX: skipping, manuals exist" + continue + elif is_tag_ignored "$repo" "$tag"; then + echo "$LOG_PREFIX: skipping, tag is ignored" + continue + fi + + clone_repo "$repo" "$tag" + build_publish_manuals "$repo" "$tag" + done +done -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/32016 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I0ecb238660553c3c857e1b310873eca8a8d09dab Gerrit-Change-Number: 32016 Gerrit-PatchSet: 3 Gerrit-Owner: osmith <[email protected]> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: laforge <[email protected]> Gerrit-Reviewer: osmith <[email protected]> Gerrit-Reviewer: pespin <[email protected]> Gerrit-MessageType: merged
