osmith has uploaded this change for review. (
https://gerrit.osmocom.org/c/osmo-ci/+/41370?usp=email )
Change subject: OBS: support building pyhss
......................................................................
OBS: support building pyhss
Build source packages for the python project pyhss similar to how we do
it for erlang projects: by downloading all dependencies ahead of time
and vendoring them in the source package.
Related: SYS#6819
Change-Id: I321090e811f7c5c142bf973c616d83cd5b2219ab
---
M scripts/obs/data/build_srcpkg.Dockerfile
A scripts/obs/data/pyhss_download_deps.sh
M scripts/obs/lib/__init__.py
M scripts/obs/lib/srcpkg.py
4 files changed, 113 insertions(+), 0 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/70/41370/1
diff --git a/scripts/obs/data/build_srcpkg.Dockerfile
b/scripts/obs/data/build_srcpkg.Dockerfile
index 8b57931..e0368fb 100644
--- a/scripts/obs/data/build_srcpkg.Dockerfile
+++ b/scripts/obs/data/build_srcpkg.Dockerfile
@@ -3,13 +3,19 @@
FROM ${DISTRO_FROM}
ARG UID
+# default-libmysqlclient-dev: needed for fetching the source package
+# "mysqlclient" with pip that PyHSS depends on. Pip actually compiles the
+# package to figure out its dependency tree and aborts if libmysqlclient-dev is
+# missing (https://github.com/pypa/pip/issues/1884).
RUN apt-get update && \
apt-get upgrade -y && \
apt-get install -y --no-install-recommends \
ca-certificates \
colordiff \
debhelper \
+ default-libmysqlclient-dev \
dh-python \
+ dh-virtualenv \
dpkg-dev \
erlang-nox \
fakeroot \
@@ -20,7 +26,9 @@
lsb-release \
meson \
osc \
+ pkgconf \
python3-packaging \
+ python3-pip \
python3-setuptools \
quilt \
sed \
diff --git a/scripts/obs/data/pyhss_download_deps.sh
b/scripts/obs/data/pyhss_download_deps.sh
new file mode 100644
index 0000000..7dfd5d9
--- /dev/null
+++ b/scripts/obs/data/pyhss_download_deps.sh
@@ -0,0 +1,97 @@
+#!/bin/sh -e
+# Copyright 2025 sysmocom - s.f.m.c. GmbH
+# SPDX-License-Identifier: GPL-3.0-or-later
+# This script downloads all dependencies of PyHSS from the python package
+# index, either as binary package for all python versions and CPU architectures
+# we care about, or as source package depending on what is available.
+
+check_cwd() {
+ if ! [ -e services/hssService.py ]; then
+ echo "ERROR: run this script from the PyHSS directory"
+ exit 1
+ fi
+ if [ -d debian/deps ]; then
+ echo "ERROR: debian/deps exists already!"
+ exit 1
+ fi
+}
+
+download_deps() {
+ local srcpkgs=_temp/requirements-source.txt
+ local binpkgs=_temp/requirements-binary.txt
+ local py_ver
+ local python_versions="
+ 3.11
+ 3.12
+ 3.13
+ "
+ # See e.g. https://pypi.org/project/SQLAlchemy/#files
+ local platform
+ local platforms="
+ manylinux_2_17_aarch64
+ manylinux_2_17_x86_64
+ "
+
+ rm -rf _temp
+ mkdir _temp
+
+ while IFS= read -r line; do
+ case "$line" in
+ # These packages are only available as sources
+ mongo*|pymongo*|mysqlclient*|pysctp*|pycryptodome=*)
+ echo "$line" >>"$srcpkgs"
+ ;;
+ # The rest is available as binary packages. This is
+ # preferred as dependencies for building some of these
+ # are not always available in the target distributions
+ # (e.g. pydantic-core is written in rust and tooling
+ # for building python + rust is not in debian 12).
+ *)
+ echo "$line" >>"$binpkgs"
+ ;;
+ esac
+ done < "requirements.txt"
+
+ # Build system dependencies must also be installed as we will build
+ # offline with --no-index and pip won't use system libraries:
+ # https://github.com/pypa/pip/issues/5696
+ echo "setuptools" >>"$binpkgs"
+ echo "wheel" >>"$binpkgs"
+ echo "hatchling" >>"$binpkgs"
+
+ echo ":: Downloading source packages"
+ pip download \
+ --dest debian/deps \
+ --no-binary=:all: \
+ -r "$srcpkgs"
+
+ for py_ver in $python_versions; do
+ for platform in $platforms; do
+ echo ":: Downloading binary packages (python $py_ver,
platform $platform)"
+ local binpkgs_extra=""
+
+ # Redis depends on async-timeout, which has been
upstreamed
+ # into Python 3.11+. This means "pip download" may not
download
+ # it if it runs with a more recent python version, but
older
+ # distros (debian 12) will need it.
+ if [ "$py_ver" = "3.11" ]; then
+ binpkgs_extra="async-timeout"
+ fi
+
+ pip download \
+ --dest debian/deps \
+ --python-version "$py_ver" \
+ --platform "$platform" \
+ --only-binary=:all: \
+ -r "$binpkgs" \
+ $binpkgs_extra
+ done
+ done
+
+ rm -r _temp
+}
+
+check_cwd
+download_deps
+
+echo ":: Success"
diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py
index 66090f2..4d0d624 100644
--- a/scripts/obs/lib/__init__.py
+++ b/scripts/obs/lib/__init__.py
@@ -16,6 +16,8 @@
# Print output of commands as they run, not only on error
cmds_verbose = False
+ci_obs_dir = os.path.realpath(f"{__file__}/../..")
+
def add_shared_arguments(parser):
""" Arguments shared between build_srcpkg.py and update_obs_project.py. """
diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py
index 78e49d1..1b31f84 100644
--- a/scripts/obs/lib/srcpkg.py
+++ b/scripts/obs/lib/srcpkg.py
@@ -141,6 +141,12 @@
)
+def prepare_project_pyhss():
+ repo_path = lib.git.get_repo_path("pyhss")
+ script = os.path.join(lib.ci_obs_dir, "data/pyhss_download_deps.sh")
+ lib.run_cmd(["sh", "-e", script], cwd=repo_path)
+
+
def run_generate_build_dep(project):
"""Run contrib/generate_build_dep.sh if it exists in the given project, to
to download sources for dependencies (see e.g. osmo_dia2gsup.git)."""
--
To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/41370?usp=email
To unsubscribe, or for help writing mail filters, visit
https://gerrit.osmocom.org/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-Change-Id: I321090e811f7c5c142bf973c616d83cd5b2219ab
Gerrit-Change-Number: 41370
Gerrit-PatchSet: 1
Gerrit-Owner: osmith <[email protected]>
