This is an automated email from the ASF dual-hosted git repository. nickva pushed a commit to branch cleanup-and-buildx-remotes-for-pkgs in repository https://gitbox.apache.org/repos/asf/couchdb-pkg.git
commit 1c4560d6ea204de43cc579b4060579b05eac6285 Author: Nick Vatamaniuc <[email protected]> AuthorDate: Mon Oct 12 01:09:52 2026 -0400 Cleanup package builds and use remote buildx nodes Cleanup some makefile targets (remove the silly redundancy), and also build package on remote buildx instances, based on architecture/os combos. That's how CI images are built. And up until now package were still built on my (host) laptop using emulation which was very slow. The setup is something like ``` docker buildx rm multiarch || true docker buildx create --name multiarch --driver docker-container --platform linux/arm64 ssh://ubuntu-nc-arm64-12 docker buildx create --append --name multiarch --driver docker-container --platform linux/amd64 ssh://ubuntu-fsn-1-09 docker buildx create --append --name multiarch --driver docker-container --platform linux/s390x ssh://linux1 docker buildx create --append --name multiarch --driver docker-container --platform linux/ppc64le ssh://ubuntu ``` Re-bootstrap after OS updates ``` docker buildx inspect --bootstrap ``` --- .dockerignore | 14 +++++++ Dockerfile | 36 +++++++++++++++++ Makefile | 121 +++++----------------------------------------------------- README.md | 57 +++++++++++++++------------ build.sh | 43 ++++++++++----------- 5 files changed, 113 insertions(+), 158 deletions(-) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..bcdd323 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Keep only the package sources and CouchDB tarball +.git +.idea +archive +pkgs +pkgs.bak* +*.bak +repo/db +repo/dists +repo/pool +parts +prime +stage +*.snap diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2c4cb4d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# check=skip=InvalidDefaultArgInFrom +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +# Build CouchDB packages for an {os,arch} combo inside the matching couchdb-ci image. +# Run from build.sh as: +# +# docker buildx build --platform linux/<arch> --output type=local,dest=pkgs/couch ... +# +# The reuslt is the built package(s). Then the local exporter saves +# them into pkgs/couch/$dist on the host. The build itself run on +# whatever node of the current builder supports combo (possibly +# remote). + +# IMAGE always passed in by build.sh. No good default for it, so added +# lint in line 1 so it stops complaining about. +ARG IMAGE +FROM ${IMAGE} AS build +ARG TARGET +ARG COUCHTARBALL +USER root +WORKDIR /home/jenkins/couchdb-pkg +COPY . . +RUN make copy-couch ${TARGET} COUCHTARBALL=${COUCHTARBALL} + +FROM scratch +COPY --from=build /home/jenkins/couchdb-pkg/pkgs/couch/ / diff --git a/Makefile b/Makefile index 06490fb..ee11b47 100644 --- a/Makefile +++ b/Makefile @@ -21,13 +21,9 @@ export [email protected] # Default package directory (over-written for RPM based builds) PKGDIR=$(COUCHDIR) -ifeq ($(shell arch),aarch64) -PKGARCH=arm64 -else ifeq ($(shell arch),arm64v8) -PKGARCH=arm64 -else +# RPMs go into the RPMS/<arch> dir where where <arch> is `uname -m` +# (x86_64, aarch64, ppc64le, ...). DEB builds figure out their own arch PKGARCH=$(shell arch) -endif # Overridden for targets with SM60 in per-target env vars below. SPIDERMONKEY=couch-libmozjs185-1.0 @@ -42,6 +38,14 @@ JAVA_RUNTIME=java21-runtime-headless | java21-runtime all: make `bin/detect-target.sh` +# arch specific prefixes arm64-ubuntu-noble are aliases for plain +# target +arm64-%: % ; @: +arm64v8-%: % ; @: +aarch64-%: % ; @: +ppc64le-%: % ; @: +s390x-%: % ; @: + # Debian default debian: sm-ver-debian find-couch-dist copy-debian update-changelog dpkg lintian copy-pkgs @@ -53,30 +57,6 @@ debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev debian-bookworm: SM_VER=78 debian-bookworm: bookworm -arm64-debian-bookworm: aarch64-debian-bookworm -arm64v8-debian-bookworm: aarch64-debian-bookworm -aarch64-debian-bookworm: PLATFORM=bookworm -aarch64-debian-bookworm: DIST=debian-bookworm -aarch64-debian-bookworm: SPIDERMONKEY=libmozjs-78-0 -aarch64-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev -aarch64-debian-bookworm: SM_VER=78 -aarch64-debian-bookworm: bookworm - -ppc64le-debian-bookworm: PLATFORM=bookworm -ppc64le-debian-bookworm: DIST=debian-bookworm -ppc64le-debian-bookworm: SPIDERMONKEY=libmozjs-78-0 -ppc64le-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev -ppc64le-debian-bookworm: SM_VER=78 -ppc64le-debian-bookworm: bookworm - -s390x-debian-bookworm: PLATFORM=bookworm -s390x-debian-bookworm: DIST=debian-bookworm -s390x-debian-bookworm: SPIDERMONKEY=libmozjs-78-0 -s390x-debian-bookworm: SPIDERMONKEY_DEV=libmozjs-78-dev -s390x-debian-bookworm: SM_VER=78 -s390x-debian-bookworm: bookworm - - bookworm: debian # Debian 13 - trixie @@ -88,32 +68,6 @@ debian-trixie: SM_VER=128 debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime debian-trixie: trixie -arm64-debian-trixie: aarch64-debian-trixie -arm64v8-debian-trixie: aarch64-debian-trixie -aarch64-debian-trixie: PLATFORM=trixie -aarch64-debian-trixie: DIST=debian-trixie -aarch64-debian-trixie: SPIDERMONKEY=libmozjs-128-0 -aarch64-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev -aarch64-debian-trixie: SM_VER=128 -aarch64-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime -aarch64-debian-trixie: trixie - -ppc64le-debian-trixie: PLATFORM=trixie -ppc64le-debian-trixie: DIST=debian-trixie -ppc64le-debian-trixie: SPIDERMONKEY=libmozjs-128-0 -ppc64le-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev -ppc64le-debian-trixie: SM_VER=128 -ppc64le-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime -ppc64le-debian-trixie: trixie - -s390x-debian-trixie: PLATFORM=trixie -s390x-debian-trixie: DIST=debian-trixie -s390x-debian-trixie: SPIDERMONKEY=libmozjs-128-0 -s390x-debian-trixie: SPIDERMONKEY_DEV=libmozjs-128-dev -s390x-debian-trixie: SM_VER=128 -s390x-debian-trixie: JAVA_RUNTIME=java21-runtime-headless | java21-runtime -s390x-debian-trixie: trixie - trixie: debian # Ubuntu 22.04 (Jammy) @@ -125,11 +79,6 @@ ubuntu-jammy: SM_VER=78 ubuntu-jammy: jammy jammy: debian -s390x-ubuntu-jammy: ubuntu-jammy -arm64-ubuntu-jammy: ubuntu-jammy -aarch64-ubuntu-jammy: ubuntu-jammy -ppc64le-ubuntu-jammy: ubuntu-jammy - # Ubuntu 24.04 (Noble) ubuntu-noble: PLATFORM=noble ubuntu-noble: DIST=ubuntu-noble @@ -139,11 +88,6 @@ ubuntu-noble: SM_VER=115 ubuntu-noble: noble noble: debian -s390x-ubuntu-noble: ubuntu-noble -arm64-ubuntu-noble: ubuntu-noble -aarch64-ubuntu-noble: ubuntu-noble -ppc64le-ubuntu-noble: ubuntu-noble - # Ubuntu 26.04 (Resolute) ubuntu-resolute: PLATFORM=resolute ubuntu-resolute: DIST=ubuntu-resolute @@ -153,11 +97,6 @@ ubuntu-resolute: SM_VER=128 ubuntu-resolute: resolute resolute: debian -s390x-ubuntu-resolute: ubuntu-resolute -arm64-ubuntu-resolute: ubuntu-resolute -aarch64-ubuntu-resolute: ubuntu-resolute -ppc64le-ubuntu-resolute: ubuntu-resolute - # RPM default centos: PKGDIR=../rpmbuild/RPMS/$(PKGARCH) centos: find-couch-dist link-couch-dist build-rpm copy-pkgs @@ -186,10 +125,6 @@ almalinux-8: centos-8 almalinux-8.8: centos-8 almalinux-8.9: centos-8 almalinux-8.10: centos-8 -aarch64-almalinux-8.10: PKGARCH=aarch64 -aarch64-almalinux-8.10: centos-8 -aarch64-almalinux-8: PKGARCH=aarch64 -aarch64-almalinux-8: centos-8 # Almalinux 9 is a CentOS 9 alias almalinux-9: centos-9 @@ -199,47 +134,11 @@ almalinux-9.5: centos-9 almalinux-9.6: centos-9 almalinux-9.7: centos-9 almalinux-9.8: centos-9 -aarch64-almalinux-9.4: PKGARCH=aarch64 -aarch64-almalinux-9.4: centos-9 -aarch64-almalinux-9.5: PKGARCH=aarch64 -aarch64-almalinux-9.5: centos-9 -aarch64-almalinux-9.6: PKGARCH=aarch64 -aarch64-almalinux-9.6: centos-9 -aarch64-almalinux-9.7: PKGARCH=aarch64 -aarch64-almalinux-9.7: centos-9 -aarch64-almalinux-9.8: PKGARCH=aarch64 -aarch64-almalinux-9.8: centos-9 -aarch64-almalinux-9: PKGARCH=aarch64 -aarch64-almalinux-9: centos-9 -# s390x RHEL 8 clone based -s390x-centos-8: centos-8 -ppc64le-centos-8: centos-8 -# s390x RHEL 9 clone based -s390x-centos-9: centos-9 - -arm64-centos-9: PKGARCH=aarch64 -arm64-centos-9: centos-9 -ppc64le-centos-9: centos-9 # Almalinux 10 is a CentOS 10 alias almalinux-10: centos-10 almalinux-10.1: centos-10 almalinux-10.2: centos-10 -aarch64-almalinux-10: PKGARCH=aarch64 -aarch64-almalinux-10: centos-10 -aarch64-almalinux-10.1: PKGARCH=aarch64 -aarch64-almalinux-10.1: centos-10 -aarch64-almalinux-10.2: PKGARCH=aarch64 -aarch64-almalinux-10.2: centos-10 -# s390x RHEL 10 clone based -s390x-centos-10: centos-10 -ppc64le-centos-10: centos-10 -# s390x RHEL 10 clone based -s390x-centos-10: centos-10 - -arm64-centos-10: PKGARCH=aarch64 -arm64-centos-10: centos-10 -ppc64le-centos-10: centos-10 # aarch64 RHEL-based aarch64-rhel: DIST=rhel diff --git a/README.md b/README.md index 999501d..9f4a938 100644 --- a/README.md +++ b/README.md @@ -41,45 +41,60 @@ make copy-couch $(lsb_release -cs) COUCHTARBALL=path/to/couchdb-#.#.#.tar.gz PLA ----- -### Building inside the `couchdbdev` docker containers +### Building inside the `couchdbci` docker containers -You must first pull down the image or images you need from Docker Hub, or build the images -using the [apache/couchdb-ci](https://github.com/apache/couchdb-ci) repository. A full -list of supported environments is at https://hub.docker.com/u/couchdbdev/ . +`build.sh` runs the package build as a `docker buildx build` with an +`apache/couchdbci-<os>:<codename>-erlang-<version>` image from +DockerHub. Those are built in the +[apache/couchdb-ci](https://github.com/apache/couchdb-ci) repo. The +image is pulled automatically and the resulting packages are written +to `pkgs/couch/<os>-<codename>/`. -#### SpiderMonkey 1.8.5 +From a downloaded CouchDB tarball: ```shell -docker pull couchdbdev/<os>-<codename>-base -./build.sh js <os>-<codename> # for example, debian-stretch, ubuntu-bionic or centos-7. +ERLANGVERSION=<erlang-version> ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz ``` -#### CouchDB +Directly from the Apache source CDN: -From a downloaded CouchDB tarball: +```shell +./build.sh couch <os>-<codename> https://dist.apache.org/repos/dist/release/couchdb/source/#.#.#/apache-couchdb-#.#.#.tar.gz +``` + +For another architecture: ```shell -docker pull couchdbdev/<osname>-<codename>-erlang-<erlang-version> -ERLANGVERSION=<erlang-version> ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz +CONTAINERARCH=arm64 ./build.sh couch <os>-<codename> path/to/couchdb-#.#.#.tar.gz ``` -Directly from the Apache source CDN: +The build is run on with buildx. With the `default` builder a foreign +arch is run under qemu (needs `binfmt_misc` handlers on the host (for +example `docker run --privileged --rm tonistiigi/binfmt --install +arm64`). To build natively use an arch specific set of builders. One +node per arch. For example: ```shell -docker pull couchdbdev/<osname>-<codename>-erlang-<erlang-version> -./build.sh couch <os>-<codename> https://dist.apache.org/repos/dist/release/couchdb/source/#.#.#/apache-couchdb-#.#.#.tar.gz +docker buildx create --name multiarch --use ssh://my-amd64-host +docker buildx create --name multiarch --append ssh://my-arm64-host ``` +Then, `buildx` will pick the node based on `--platform`. Use +`BUILDKIT_PROGRESS=auto` for a less noisy buildx progress view. + ----- ## Building packages for a release ### Prerequisites -1. Linux running Docker -1. The current user must be capable of running `docker run`. -1. Enough free disk space to download all of the Docker images + build - CouchDB. +1. Linux running Docker with buildx. +1. User must be able of running `docker buildx build`. +1. For non-x86_64 packages, need either qemu `binfmt_misc` handlers, + or buildx builders with native nodes (see above). +1. Enough free disk space on the builder to download all of the Docker + images + build CouchDB. Build layers stay in the builder's + cache. But can use `docker buildx prune` to clean up if needed. ### Running the package build @@ -98,18 +113,12 @@ or Packages will be placed in the `pkgs/couch` subdirectory. -A similar `js-all` target exists, should the SpiderMonkey packages need to be regenerated. - ### Uploading the packages If you have Apache credentials (set your `BINARY_CREDS` environment variable appropriately), after building all CouchDB packages above, **and signing the rpms with the appropriate GPG key using the `rpmsign --addsign <file.rpm>` command**, simply run: ./build.sh couch-upload-all -Or, for the SpiderMonkey packages: - - ./build.sh js-upload-all - ----- ## Upload Dev Packages diff --git a/build.sh b/build.sh index 8501ba0..f3662ac 100755 --- a/build.sh +++ b/build.sh @@ -17,12 +17,15 @@ # specific language governing permissions and limitations # under the License. -# This is the master shell script to build Docker containers -# for CouchDB 2.x. +# This is the master shell script to build CouchDB packages inside the +# couchdb-ci Docker images. # stop on error set -e +# Full build log. Can set BUILDKIT_PROGRESS=auto for a less noisy log. +export BUILDKIT_PROGRESS=${BUILDKIT_PROGRESS:-plain} + # This works if we're not called through a symlink # otherwise, see https://stackoverflow.com/questions/59895/ SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" @@ -81,26 +84,19 @@ get-couch-tarball() { } build-couch() { + # $1 - platform, for example: ubuntu-noble + # CONTAINERARCH - arch, for example (amd64) split-os-ver $1 - # We will be changing user to 'jenkins' - ensure it has write permissions - chmod a+rwx pkgs pkgs/couch pkgs/js - # $1 is plat, $2 is the optional path to a dist tarball - if [ -z ${CONTAINERARCH+x} ]; then - docker run \ - --mount type=bind,src=${SCRIPTPATH},dst=/home/jenkins/couchdb-pkg \ - -u 0 -w /home/jenkins/couchdb-pkg \ - --platform linux/amd64 \ - apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \ - make copy-couch $1 COUCHTARBALL=${COUCHTARBALL} - else - docker run \ - --mount type=bind,src=${SCRIPTPATH},dst=/home/jenkins/couchdb-pkg \ - -u 0 -w /home/jenkins/couchdb-pkg \ - --platform linux/${CONTAINERARCH} \ - apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \ - make copy-couch ${CONTAINERARCH}-$1 COUCHTARBALL=${COUCHTARBALL} - fi - make clean + local arch=${CONTAINERARCH:-amd64} + echo " **** building $1 for ${arch} **** " + docker buildx build \ + --platform linux/${arch} \ + --pull \ + --build-arg IMAGE=apache/couchdbci-${os}:${version}-erlang-${ERLANGVERSION} \ + --build-arg TARGET=$1 \ + --build-arg COUCHTARBALL=${COUCHTARBALL} \ + --output type=local,dest=${SCRIPTPATH}/pkgs/couch \ + ${SCRIPTPATH} } build-all-couch() { @@ -119,8 +115,6 @@ build-all-couch() { ( ${base} == "centos-8" && ${arch} == "ppc64le" ) || ( ${base} == "centos-9" && ${arch} == "ppc64le" ) || ( ${base} == "centos-10" && ${arch} == "ppc64le") || - ( ${base} == "debian-bullseye" && ${arch} == "ppc64le" ) || - ( ${base} == "debian-bullseye" && ${arch} == "s390x") || ( ${base} == "debian-bookworm" && ${arch} == "ppc64le" ) || ( ${base} == "debian-trixie" && ${arch} == "ppc64le" ) || ( ${base} == "ubuntu-jammy" && ${arch} == "ppc64le" ) || @@ -226,6 +220,9 @@ Recognized commands: - a URL to http(s)://domain.com/to/couchdb.tar.gz Commands marked with * require BINARY_CREDS env var. + + Set CONTAINERARCH=arm64 to build a single platform for another arch (arm64). + couch-all builds every architecture in XPLAT_ARCHES. EOF }
