Hello community, here is the log from the commit of package charliecloud for openSUSE:Factory checked in at 2020-10-21 14:40:03 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/charliecloud (Old) and /work/SRC/openSUSE:Factory/.charliecloud.new.3486 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "charliecloud" Wed Oct 21 14:40:03 2020 rev:15 rq:843193 version:0.20 Changes: -------- --- /work/SRC/openSUSE:Factory/charliecloud/charliecloud.changes 2020-09-22 21:15:25.552152224 +0200 +++ /work/SRC/openSUSE:Factory/.charliecloud.new.3486/charliecloud.changes 2020-10-21 14:40:18.061670338 +0200 @@ -1,0 +2,10 @@ +Wed Oct 21 08:14:06 UTC 2020 - Ana Guerrero Lopez <[email protected]> + +- Update to version 0.20. + - improvement for unprivileged image build using fakeroot + This feature can be turned off with “ch-grow build --no-fakeroot” + Further details are in the ch-grow man page + - miscellaneous bug fixes and improvements + - Full changelot at https://groups.io/g/charliecloud/message/107 +- Add requirements on fakeroot +------------------------------------------------------------------- Old: ---- charliecloud-0.19.tar.gz New: ---- charliecloud-0.20.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ charliecloud.spec ++++++ --- /var/tmp/diff_new_pack.5iV5Wn/_old 2020-10-21 14:40:18.965670849 +0200 +++ /var/tmp/diff_new_pack.5iV5Wn/_new 2020-10-21 14:40:18.965670849 +0200 @@ -17,7 +17,7 @@ Name: charliecloud -Version: 0.19 +Version: 0.20 Release: 0 Summary: User-defined software stacks (UDSS) for HPC centers License: Apache-2.0 @@ -28,6 +28,7 @@ BuildRequires: python3-base # Recommend for ch-grow # used to build images +Requires: fakeroot Recommends: docker Recommends: buildah >= 1.11.2 Recommends: python3-requests >= 2.6.0 ++++++ charliecloud-0.19.tar.gz -> charliecloud-0.20.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/VERSION new/charliecloud-0.20/VERSION --- old/charliecloud-0.19/VERSION 2020-09-21 22:12:11.000000000 +0200 +++ new/charliecloud-0.20/VERSION 2020-10-20 18:46:57.000000000 +0200 @@ -1 +1 @@ -0.19 +0.20 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/bin/ch-grow.py.in new/charliecloud-0.20/bin/ch-grow.py.in --- old/charliecloud-0.19/bin/ch-grow.py.in 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/bin/ch-grow.py.in 2020-10-20 18:44:46.000000000 +0200 @@ -15,22 +15,18 @@ ## Constants ## # FIXME: It's currently easy to get the ch-run path from another script, but -# hard from something in lib. So, despite the fact that only build needs this, -# we set it here for now. -build.CH_BIN = os.path.dirname(os.path.abspath( +# hard from something in lib. So, we set it here for now. +ch.CH_BIN = os.path.dirname(os.path.abspath( inspect.getframeinfo(inspect.currentframe()).filename)) -build.CH_RUN = build.CH_BIN + "/ch-run" +ch.CH_RUN = ch.CH_BIN + "/ch-run" ## Main ## def main(): - if (not os.path.exists(build.CH_RUN)): - ch.depfails.append(("missing", build.CH_RUN)) - - # https://stackoverflow.com/a/5464440 - #HF = lambda prog: argparse.HelpFormatter(prog, max_help_position=26) + if (not os.path.exists(ch.CH_RUN)): + ch.depfails.append(("missing", ch.CH_RUN)) ap = argparse.ArgumentParser(formatter_class=ch.HelpFormatter, description="Build and manage images; completely unprivileged.", @@ -116,6 +112,8 @@ help="Dockerfile to use (default: CONTEXT/Dockerfile)") sp.add_argument("-n", "--dry-run", action="store_true", help="don't execute instructions") + sp.add_argument("--no-fakeroot", action="store_true", + help="don't try unprivileged build workarounds") sp.add_argument("--parse-only", action="store_true", help="stop after parsing the Dockerfile") sp.add_argument("-t", "--tag", metavar="TAG", diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/bin/ch-run-oci.py.in new/charliecloud-0.20/bin/ch-run-oci.py.in --- old/charliecloud-0.19/bin/ch-run-oci.py.in 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/bin/ch-run-oci.py.in 2020-10-20 18:44:46.000000000 +0200 @@ -136,7 +136,6 @@ # unprivileged user namespace. See also the default environment. # # Debian apt/dpkg/etc. want to chown(1), chgrp(1), etc. in various ways. - ch.symlink("/bin/true", "%s/ch/bin/chown" % path) ch.symlink("/bin/true", "%s/ch/bin/chgrp" % path) ch.symlink("/bin/true", "%s/ch/bin/dpkg-statoverride" % path) # Debian package management also wants to mess around with users. This is @@ -144,6 +143,7 @@ # work if they are in /ch/bin, I think because dpkg is resetting the path? # For now we'll do this, but I don't like it. fakeroot(1) also solves the # problem (see issue #472). + ch.symlink("/bin/true", "%s/bin/chown" % path, clobber=True) ch.symlink("/bin/true", "%s/usr/sbin/groupadd" % path, clobber=True) ch.symlink("/bin/true", "%s/usr/sbin/useradd" % path, clobber=True) ch.symlink("/bin/true", "%s/usr/sbin/usermod" % path, clobber=True) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/configure new/charliecloud-0.20/configure --- old/charliecloud-0.19/configure 2020-09-21 22:12:29.000000000 +0200 +++ new/charliecloud-0.20/configure 2020-10-20 18:47:14.000000000 +0200 @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Charliecloud 0.19. +# Generated by GNU Autoconf 2.69 for Charliecloud 0.20. # # Report bugs to <https://github.com/hpc/charliecloud>. # @@ -580,8 +580,8 @@ # Identity of this package. PACKAGE_NAME='Charliecloud' PACKAGE_TARNAME='charliecloud' -PACKAGE_VERSION='0.19' -PACKAGE_STRING='Charliecloud 0.19' +PACKAGE_VERSION='0.20' +PACKAGE_STRING='Charliecloud 0.20' PACKAGE_BUGREPORT='https://github.com/hpc/charliecloud' PACKAGE_URL='' @@ -1301,7 +1301,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Charliecloud 0.19 to adapt to many kinds of systems. +\`configure' configures Charliecloud 0.20 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1372,7 +1372,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Charliecloud 0.19:";; + short | recursive ) echo "Configuration of Charliecloud 0.20:";; esac cat <<\_ACEOF @@ -1480,7 +1480,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Charliecloud configure 0.19 +Charliecloud configure 0.20 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1660,7 +1660,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Charliecloud $as_me 0.19, which was +It was created by Charliecloud $as_me 0.20, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2608,7 +2608,7 @@ # Define the identity of the package. PACKAGE='charliecloud' - VERSION='0.19' + VERSION='0.20' cat >>confdefs.h <<_ACEOF @@ -6196,7 +6196,7 @@ { $as_echo "$as_me:${as_lineno-$LINENO}: checking for nVidia libraries & executables" >&5 $as_echo_n "checking for nVidia libraries & executables... " >&6; } -if test -n "$NVIDIA_CLI_VERSION"; then : +if test -n "$NVIDIA_CLI"; then : if nvidia-container-cli list | grep -Fq libnvidia-glcore.so; then : have_nvidia_libs=yes else @@ -6294,7 +6294,7 @@ esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking if $PYTHON version >= $vmin_python" >&5 $as_echo_n "checking if $PYTHON version >= $vmin_python... " >&6; } - vact=$($PYTHON --version | cut -d' ' -f2) + vact=$($PYTHON --version | head -1 | cut -d' ' -f2) @@ -7761,7 +7761,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Charliecloud $as_me 0.19, which was +This file was extended by Charliecloud $as_me 0.20, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -7827,7 +7827,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Charliecloud config.status 0.19 +Charliecloud config.status 0.20 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/configure.ac new/charliecloud-0.20/configure.ac --- old/charliecloud-0.19/configure.ac 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/configure.ac 2020-10-10 00:44:26.000000000 +0200 @@ -287,7 +287,7 @@ CH_CHECK_VERSION([NVIDIA_CLI], [$vmin_nvidia_CLI], [-V | head -1 | cut -d' ' -f2]) AC_MSG_CHECKING([for nVidia libraries & executables]) -AS_IF([test -n "$NVIDIA_CLI_VERSION"], +AS_IF([test -n "$NVIDIA_CLI"], [AS_IF([nvidia-container-cli list | grep -Fq libnvidia-glcore.so], [have_nvidia_libs=yes], [have_nvidia_libs=no])], @@ -307,7 +307,8 @@ [/*], [PYTHON="$python"], # absolute [*], [AC_CHECK_PROG([PYTHON], [$python], [$python])] # verify it's in $PATH ) -CH_CHECK_VERSION([PYTHON], [$vmin_python], [--version | cut -d' ' -f2]) +CH_CHECK_VERSION([PYTHON], [$vmin_python], + [--version | head -1 | cut -d' ' -f2]) # Python module "lark-parser" vmin_lark=0.7.1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/doc/ch-grow_desc.rst new/charliecloud-0.20/doc/ch-grow_desc.rst --- old/charliecloud-0.19/doc/ch-grow_desc.rst 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/doc/ch-grow_desc.rst 2020-10-20 18:44:46.000000000 +0200 @@ -108,7 +108,11 @@ -f -` behavior. :code:`-n`, :code:`--dry-run` - Do not actually execute any Dockerfile instructions. + Don't actually execute any Dockerfile instructions. + + :code:`--no-fakeroot` + Don't try any of the unprivileged build workarounds (see section "Quirks + of a fully unprivileged builds" below). :code:`--parse-only` Stop after parsing the Dockerfile. @@ -151,6 +155,65 @@ without talking to the internet or touching the storage directory. +Quirks of a fully unprivileged build +==================================== + +:code:`ch-grow` is *fully* unprivileged. It runs all instructions as the +normal user who invokes it, does not use any setuid or setcap helper programs, +and does not use :code:`/etc/subuid` or :code:`/etc/subgid`, in contrast to +the “rootless” mode of some competing builders. This is accomplished by +executing :code:`RUN` instructions with :code:`ch-run -w --uid=0 --gid=0` (and +some other arguments), i.e., your host EUID and EGID both mapped to zero +inside the container, and only one UID (zero) and GID (zero) are available +inside the container. + +Under this arrangement, processes running in the container *appear* to be +running as root, but many privileged system calls will fail without the +workarounds described below. **This affects any fully unprivileged +container build, not just Charliecloud.** + +The most common time to see this is installing packages. For example, here is +RPM failing to :code:`chown(2)` a file, which makes the package update fail: + +.. code-block:: none + + Updating : 1:dbus-1.10.24-13.el7_6.x86_64 2/4 + Error unpacking rpm package 1:dbus-1.10.24-13.el7_6.x86_64 + error: unpacking of archive failed on file /usr/libexec/dbus-1/dbus-daemon-launch-helper;5cffd726: cpio: chown + Cleanup : 1:dbus-libs-1.10.24-12.el7.x86_64 3/4 + error: dbus-1:1.10.24-13.el7_6.x86_64: install failed + +This one is (ironically) :code:`apt-get` failing to drop privileges: + +.. code-block:: none + + E: setgroups 65534 failed - setgroups (1: Operation not permitted) + E: setegid 65534 failed - setegid (22: Invalid argument) + E: seteuid 100 failed - seteuid (22: Invalid argument) + E: setgroups 0 failed - setgroups (1: Operation not permitted) + +The solution :code:`ch-grow` uses is to intercept these system calls and fake +a successful result. We accomplish this by altering the Dockerfile to call +:code:`fakeroot(1)` (of which there are several implementations) for +:code:`RUN` instructions that seem to need it. There are two basic steps: + + 1. After :code:`FROM`, install a :code:`fakeroot(1)` implementation. This + sometimes also needs extra steps like turning off the :code:`apt` sandbox + (for Debian Buster) or enabling EPEL (for CentOS/RHEL). + + 2. Prepend :code:`fakeroot` to :code:`RUN` instructions that seem to need + it, e.g. ones that contain :code:`apt`, :code:`apt-get`, :code:`dpkg` for + Debian derivatives and :code:`dnf`, :code:`rpm`, or :code:`yum` for + RPM-based distributions. + +The details are specific to each distribution. :code:`ch-grow` analyzes image +content (e.g., grepping :code:`/etc/debian_version`) to select a +configuration; see :code:`lib/fakeroot.py` for details. :code:`ch-grow` prints +exactly what it is doing. + +To turn off this behavior, use the :code:`--no-fakeroot` option. + + Compatibility with other Dockerfile interpreters ================================================ @@ -182,37 +245,6 @@ assessments and open questions. This helps us prioritize new features and revise our thinking about what is needed for HPC containers. -Quirks of a fully unprivileged build ------------------------------------- - -:code:`ch-grow` is *fully* unprivileged. It runs all instructions as the -normal user who invokes it, does not use any setuid or setcap helper programs, -and does not use :code:`/etc/subuid` or :code:`/etc/subgid`, in contrast to -the “rootless” mode of some competing builders. - -:code:`RUN` instructions are executed with :code:`ch-run --uid=0 --gid=0`, -i.e., host EUID and EGID both mapped to zero inside the container, and only -one UID (zero) and GID (zero) are available inside the container. Also, -:code:`/etc/passwd` and :code:`/etc/group` are bind-mounted from temporary -files outside the container and can't be written. (Strictly speaking, the -files themselves are read-write, but because they are bind-mounted, the common -pattern of writing a new file and moving it on top of the existing one fails.) - -This has two consequences: the shell and its children appear to be running as -root but only some privileged system calls are available, and manipulating -users and groups will fail. This confuses some programs, which fail with -"permission denied" and related errors; for example, :code:`chgrp(1)` often -appears in Debian package post-install scripts. We have worked around some of -these problems, but many remain. Another manual workaround is to install -:code:`fakeroot` in the Dockerfile and prepend :code:`fakeroot` to problem -commands. - -.. note:: - - Most of these issues affect *any* fully unprivileged container build, not - just :code:`ch-grow`. We are working to better characterize the problems - and add automatic workarounds. - Context directory ----------------- diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/examples/Dockerfile.centos7 new/charliecloud-0.20/examples/Dockerfile.centos7 --- old/charliecloud-0.19/examples/Dockerfile.centos7 2020-08-11 00:11:18.000000000 +0200 +++ new/charliecloud-0.20/examples/Dockerfile.centos7 2020-10-20 18:44:46.000000000 +0200 @@ -4,29 +4,15 @@ # This image has two purposes: (1) demonstrate we can build a CentOS 7 image # and (2) provide a build environment for Charliecloud EPEL 7 RPMs. -# Re. ch-grow: Like apt(8), Yum/RPM like to chown(2), etc., if they believe they -# are root. This fails in an unprivileged user namespace because UID 0 is fake. -# Unlike apt, RPM makes these system calls directly, so there's no opportunity -# for kludges like linking chown(1) to true(1). For example: -# -# Updating : 1:dbus-1.10.24-13.el7_6.x86_64 2/4 -# Error unpacking rpm package 1:dbus-1.10.24-13.el7_6.x86_64 -# error: unpacking of archive failed on file /usr/libexec/dbus-1/dbus-daemon-launch-helper;5cffd726: cpio: chown -# Cleanup : 1:dbus-libs-1.10.24-12.el7.x86_64 3/4 -# error: dbus-1:1.10.24-13.el7_6.x86_64: install failed -# error: dbus-1:1.10.24-12.el7.x86_64: erase skipped -# -# We can instead use fakeroot(1), though there seems to be a performance -# impact. In the interest of time, we demonstrate this by installing the -# openssh package, required by git, which reliably tickles the problem. -# See issue #472. +# Install our dependencies, ensuring we fail out if any are missing. RUN yum install -y epel-release \ - && yum install -y \ + && yum install -y --setopt=skip_missing_names_on_install=0 \ autoconf \ automake \ bats \ fakeroot \ gcc \ + git \ make \ python36 \ python36-sphinx \ @@ -35,7 +21,4 @@ rpmlint \ rsync \ wget \ - && fakeroot yum install -y \ - git \ - openssh \ && yum clean all diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/examples/Dockerfile.centos8 new/charliecloud-0.20/examples/Dockerfile.centos8 --- old/charliecloud-0.19/examples/Dockerfile.centos8 2020-08-11 00:11:18.000000000 +0200 +++ new/charliecloud-0.20/examples/Dockerfile.centos8 2020-10-10 00:44:26.000000000 +0200 @@ -3,31 +3,17 @@ # This image has two purposes: (1) demonstrate we can build a CentOS 8 image # and (2) provide avbuild environment for Charliecloud EPEL 8 RPMs. - -# Re. ch-grow: Like apt(8), dnf/RPM like to chown(2), etc., if they believe they -# are root. This fails in an unprivileged user namespace because UID 0 is fake. -# Unlike apt, RPM makes these system calls directly, so there's no opportunity -# for kludges like linking chown(1) to true(1). For example: # -# Updating : 1:dbus-1.10.24-13.el7_6.x86_64 2/4 -# Error unpacking rpm package 1:dbus-1.10.24-13.el7_6.x86_64 -# error: unpacking of archive failed on file /usr/libexec/dbus-1/dbus-daemon-launch-helper;5cffd726: -# Cleanup : 1:dbus-libs-1.10.24-12.el7.x86_64 3/4 -# error: dbus-1:1.10.24-13.el7_6.x86_64: install failed -# error: dbus-1:1.10.24-12.el7.x86_64: erase skipped +# Quirks: # -# We can instead use fakeroot(1), though there seems to be a performance -# impact. In the interest of time, we demonstrate this by installing the -# openssh package, required by git, which reliably tickles the problem. -# See issue #472. - -# 1. Install the dnf ovl plugin to work around RPMDB corruption when building -# images with Docker and the OverlayFS storage driver. +# 1. Install the dnf ovl plugin to work around RPMDB corruption when +# building images with Docker and the OverlayFS storage driver. # -# 2. Enable PowerTools repository, as some packages in EPEL depend on it. Use -# sed(1) because we don't want to install `dnf-plugins-core` just for this. +# 2. Enable PowerTools repository, because some packages in EPEL depend on +# it. Use sed(1) because we don't want to install `dnf-plugins-core` just +# for this. # -# 3. Install packages needed to build el8 rpms. +# 3. Install packages needed to build el8 rpms. # RUN dnf install -y --setopt=install_weak_deps=false epel-release \ && sed -ie 's/enabled=0/enabled=1/' /etc/yum.repos.d/CentOS-PowerTools.repo \ @@ -36,6 +22,7 @@ autoconf \ automake \ gcc \ + git \ make \ python3 \ python3-sphinx \ @@ -44,10 +31,6 @@ rpmlint \ rsync \ wget \ - fakeroot \ - && fakeroot dnf install -y --setopt=install_weak_deps=false \ - git \ - openssh \ && dnf clean all # CentOS's linker doesn't search these paths by default; add them because we diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/examples/spack/Dockerfile new/charliecloud-0.20/examples/spack/Dockerfile --- old/charliecloud-0.19/examples/spack/Dockerfile 2020-08-11 00:11:18.000000000 +0200 +++ new/charliecloud-0.20/examples/spack/Dockerfile 2020-10-20 18:44:46.000000000 +0200 @@ -1,5 +1,5 @@ # ch-test-scope: full -FROM centos:7 +FROM centos:8 # Note: Spack is a bit of an odd duck testing wise. Because it's a package # manager, the key tests we want are to install stuff (this includes the Spack @@ -15,25 +15,22 @@ # Packages needed to install Spack [1]. # -# Note: Spack claims that Python 3 works, but using python3 here fails later -# with "/usr/bin/env: 'python': No such file or directory". -# bzip, file, patch, unzip, and which are packages needed to install +# bzip, file, patch, unzip, and which are packages needed to install # Charliecloud with Spack. These are in Spack's Docker example [2] but are not # documented as prerequisites [1]. -RUN yum install -y \ - curl \ +RUN dnf install -y --setopt=install_weak_deps=false \ gcc \ gcc-c++ \ git \ gnupg2-smime \ - python \ + python3 \ make \ bzip2 \ file \ patch \ unzip \ which \ - && yum clean all + && dnf clean all # Certain Spack packages (e.g., tar) puke if they detect themselves being # configured as UID 0. This is the override. See issue #540 and [2]. @@ -44,14 +41,9 @@ # place it at a standard path ("spack clone" simply clones another working # directory to a new path). # -# Spack does have releases, but they seem pretty stale. As of 2019-09-12, the -# most recent version is 0.12.1 dated 8 months ago, 2019-01-13; there have -# been hundreds if not thousands of commits on the default branch "develop" -# since then. We follow develop for this reason and also to catch problems -# installing Charliecloud with latest Spack. +# We follow the develop branch to catch problems installing Charliecloud with +# the latest Spack. ARG SPACK_REPO=https://github.com/spack/spack -#ENV SPACK_VERSION 0.12.1 -#RUN git clone --branch v$SPACK_VERSION --depth 1 $SPACK_REPO RUN git clone --depth 1 $SPACK_REPO RUN cd spack && git status && git rev-parse --short HEAD diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/examples/spark/Dockerfile new/charliecloud-0.20/examples/spark/Dockerfile --- old/charliecloud-0.19/examples/spark/Dockerfile 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/examples/spark/Dockerfile 2020-10-20 18:44:46.000000000 +0200 @@ -1,12 +1,18 @@ # ch-test-scope: standard -FROM debian:stretch +# Use Buster because Stretch JRE install fails with: +# +# tempnam() is so ludicrously insecure as to defy implementation.tempnam: Cannot allocate memory +# dpkg: error processing package openjdk-8-jre-headless:amd64 (--configure): +# subprocess installed post-installation script returned error exit status 1 + +FROM debian:buster ARG DEBIAN_FRONTEND=noninteractive # Install needed OS packages. RUN apt-get update \ && apt-get install -y --no-install-recommends \ + default-jre-headless \ less \ - openjdk-8-jre-headless \ procps \ python \ wget \ @@ -24,8 +30,8 @@ # # 3. We disapprove of Spark's master/slave terminology, but it's what the # scripts are called, so we don't see a way to avoid it currently. -ARG URLPATH=https://archive.apache.org/dist/spark/spark-2.4.7/ -ARG DIR=spark-2.4.7-bin-hadoop2.7 +ARG URLPATH=https://archive.apache.org/dist/spark/spark-3.0.1/ +ARG DIR=spark-3.0.1-bin-hadoop2.7 ARG TAR=$DIR.tgz RUN wget -nv $URLPATH/$TAR \ && tar xf $TAR \ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/lib/Makefile.am new/charliecloud-0.20/lib/Makefile.am --- old/charliecloud-0.19/lib/Makefile.am 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/lib/Makefile.am 2020-10-10 00:44:26.000000000 +0200 @@ -5,7 +5,7 @@ # See: https://www.gnu.org/software/automake/manual/html_node/Uniform.html mylibdir = $(pkglibdir) -dist_mylib_DATA = base.sh build.py charliecloud.py misc.py +dist_mylib_DATA = base.sh build.py fakeroot.py charliecloud.py misc.py noinst_DATA = charliecloud mylib_DATA = contributors.bash version.py version.sh version.txt diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/lib/Makefile.in new/charliecloud-0.20/lib/Makefile.in --- old/charliecloud-0.19/lib/Makefile.in 2020-09-21 22:12:30.000000000 +0200 +++ new/charliecloud-0.20/lib/Makefile.in 2020-10-20 18:47:14.000000000 +0200 @@ -292,7 +292,7 @@ # # See: https://www.gnu.org/software/automake/manual/html_node/Uniform.html mylibdir = $(pkglibdir) -dist_mylib_DATA = base.sh build.py charliecloud.py misc.py +dist_mylib_DATA = base.sh build.py fakeroot.py charliecloud.py misc.py noinst_DATA = charliecloud mylib_DATA = contributors.bash version.py version.sh version.txt CLEANFILES = $(mylib_DATA) $(noinst_DATA) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/lib/build.py new/charliecloud-0.20/lib/build.py --- old/charliecloud-0.19/lib/build.py 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/lib/build.py 2020-10-10 00:44:26.000000000 +0200 @@ -11,6 +11,7 @@ import sys import charliecloud as ch +import fakeroot ## Globals ## @@ -41,10 +42,6 @@ ## Constants ## -# FIXME: currently set in ch-grow :P -CH_BIN = None -CH_RUN = None - ARG_DEFAULTS = { "HTTP_PROXY": os.environ.get("HTTP_PROXY"), "HTTPS_PROXY": os.environ.get("HTTPS_PROXY"), "FTP_PROXY": os.environ.get("FTP_PROXY"), @@ -53,7 +50,7 @@ "https_proxy": os.environ.get("https_proxy"), "ftp_proxy": os.environ.get("ftp_proxy"), "no_proxy": os.environ.get("no_proxy"), - "PATH": "/ch/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", # GNU tar, when it thinks it's running as root, tries to # chown(2) and chgrp(2) files to whatever's in the tarball. "TAR_OPTIONS": "--no-same-owner" } @@ -532,6 +529,9 @@ self.base_image.pull_to_unpacked(fixup=True) image.copy_unpacked(self.base_image) env.reset() + # Inject fakeroot preparatory stuff if needed. + if (not cli.no_fakeroot): + fakeroot.inject_first(image.unpack_path, env.env_build) def str_(self): alias = "AS %s" % self.alias if self.alias else "" @@ -540,14 +540,17 @@ class Run(Instruction): + def cmd_set(self, args): + # This can be called if RUN is erroneously placed before FROM; in this + # case there is no image yet, so don't inject. + if (cli.no_fakeroot or image_i not in images): + self.cmd = args + else: + self.cmd = fakeroot.inject_each(images[image_i].unpack_path, args) + def execute_(self): rootfs = images[image_i].unpack_path - ch.file_ensure_exists(rootfs + "/etc/resolv.conf") - ch.file_ensure_exists(rootfs + "/etc/hosts") - args = [CH_BIN + "/ch-run", "-w", "--no-home", "--no-passwd", - "--cd", env.workdir, "--uid=0", "--gid=0", - rootfs, "--"] + self.cmd - ch.cmd(args, env=env.env_build) + ch.ch_run_modify(rootfs, self.cmd, env.env_build, env.workdir) def str_(self): return str(self.cmd) @@ -557,8 +560,8 @@ def __init__(self, *args): super().__init__(*args) - self.cmd = [ variables_sub(unescape(i), env.env_build) - for i in ch.tree_terminals(self.tree, "STRING_QUOTED")] + self.cmd_set([ variables_sub(unescape(i), env.env_build) + for i in ch.tree_terminals(self.tree, "STRING_QUOTED")]) class I_run_shell(Run): @@ -567,7 +570,7 @@ super().__init__(*args) # FIXME: Can't figure out how to remove continuations at parse time. cmd = ch.tree_terminal(self.tree, "LINE").replace("\\\n", "") - self.cmd = ["/bin/sh", "-c", cmd] + self.cmd_set(["/bin/sh", "-c", cmd]) class I_workdir(Instruction): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/lib/charliecloud.py new/charliecloud-0.20/lib/charliecloud.py --- old/charliecloud-0.19/lib/charliecloud.py 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/lib/charliecloud.py 2020-10-20 18:44:46.000000000 +0200 @@ -55,6 +55,10 @@ ## Globals ## +# FIXME: currently set in ch-grow :P +CH_BIN = None +CH_RUN = None + # Logging; set using log_setup() below. verbose = 0 # Verbosity level. Can be 0, 1, or 2. log_festoon = False # If true, prepend pid and timestamp to chatter. @@ -220,8 +224,8 @@ def copy_unpacked(self, other): "Copy the unpack directory of Image other to my unpack directory." - DEBUG("copying image: %s -> %s" % (other.unpack_path, self.unpack_path)) self.unpack_create_ok() + DEBUG("copying image: %s -> %s" % (other.unpack_path, self.unpack_path)) copytree(other.unpack_path, self.unpack_path, symlinks=True) def download(self, use_cache): @@ -256,27 +260,11 @@ "Add the Charliecloud workarounds to the unpacked image." DEBUG("fixing up image: %s" % self.unpack_path) # Metadata directory. - mkdirs("%s/ch/bin" % self.unpack_path) + mkdirs("%s/ch" % self.unpack_path) file_ensure_exists("%s/ch/environment" % self.unpack_path) # Mount points. file_ensure_exists("%s/etc/hosts" % self.unpack_path) file_ensure_exists("%s/etc/resolv.conf" % self.unpack_path) - # /etc/{passwd,group} - file_write("%s/etc/passwd" % self.unpack_path, """\ -root:x:0:0:root:/root:/bin/sh -nobody:x:65534:65534:nobody:/:/bin/false -""") - file_write("%s/etc/group" % self.unpack_path, """\ -root:x:0: -nogroup:x:65534: -""") - # Kludges to work around expectations of real root, not UID 0 in a - # unprivileged user namespace. See also the default environment. - # - # Debian "apt" and friends want to chown(1), chgrp(1), etc. - symlink("/bin/true", "%s/ch/bin/chown" % self.unpack_path) - symlink("/bin/true", "%s/ch/bin/chgrp" % self.unpack_path) - symlink("/bin/true", "%s/ch/bin/dpkg-statoverride" % self.unpack_path) def flatten(self): "Flatten the layers in the download cache into the unpack directory." @@ -832,6 +820,11 @@ def WARNING(*args, **kwargs): log(color="31m", prefix="warning: ", *args, **kwargs) +def ch_run_modify(img, args, env, workdir="/"): + args = [CH_BIN + "/ch-run", "-w", "--cd", workdir, "--uid=0", "--gid=0", + "--no-home", "--no-passwd", img, "--"] + args + cmd(args, env) + def cmd(args, env=None): DEBUG("environment: %s" % env) DEBUG("executing: %s" % args) @@ -876,6 +869,19 @@ ossafe(os.chmod, "can't chmod 0%o: %s" % (mode, path)) fp.close() +def grep_p(path, rx): + """Return True if file at path contains a line matching regular expression + rx, False if it does not.""" + rx = re.compile(rx) + try: + with open(path, "rt") as fp: + for line in fp: + if (rx.search(line) is not None): + return True + return False + except OSError as x: + FATAL("error reading %s: %s" % (path, x.strerror)) + def log(*args, color=None, prefix="", **kwargs): if (color is not None): color_set(color, log_fp) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/lib/fakeroot.py new/charliecloud-0.20/lib/fakeroot.py --- old/charliecloud-0.19/lib/fakeroot.py 1970-01-01 01:00:00.000000000 +0100 +++ new/charliecloud-0.20/lib/fakeroot.py 2020-10-10 00:44:26.000000000 +0200 @@ -0,0 +1,130 @@ +import os.path + +import charliecloud as ch + + +## Globals ## + +# FIXME: document this config +# FIXME: sequence of command vs. one long command? + +DEFAULT_CONFIGS = [ + + # General notes: + # + # 1. The first match here wins. + # + # 2. There are three implementations of fakeroot that I could find: + # fakeroot, fakeroot-ng, and pseudo. As of 2020-09-02: + # + # * fakeroot-ng and pseudo use a daemon process, while fakeroot does + # not. pseudo also uses a persistent database. + # + # * fakeroot-ng does not support ARM; pseudo supports many architectures + # including ARM. + # + # * “Old” fakeroot seems to have had version 1.24 on 2019-09-07 with + # the most recent commit 2020-08-12. + # + # * fakeroot-ng is quite old: last upstream release was 0.18 in 2013, + # and its source code is on Sourceforge. + # + # * pseudo is aslo a bit old: last upstream version was 1.9.0 on + # 2018-01-20, and the last Git commit was 2019-08-02. + # + # Generally, we select the first one that seems to work in the order + # fakeroot, pseudo, fakeroot-ng. + # + # 3. Why grep specified files vs. simpler alternatives? + # + # * Look at image name: Misses derived images, large number of tags + # seems a maintenance headache, :latest changes. + # + # * grep the same file for each distro: No standardized file for this. + # + # * Ask lsb_release(1): Not always installed, requires executing ch-run. + + # CentOS/RHEL notes: + # + # 1. CentOS seems to have only fakeroot, which is in EPEL, not the standard + # repos. + + { "match": ("/etc/redhat-release", r"release 7\."), + "config": { "name": "CentOS/RHEL 7", + "first": ["yum install -y epel-release", + "yum install -y fakeroot"], + "cmds_each": ["dnf", "rpm", "yum"], + "each": ["fakeroot"] } }, + { "match": ("/etc/redhat-release", r"release 8\."), + "config": { "name": "CentOS/RHEL 8", + "first": ["dnf install -y epel-release", + "dnf install -y fakeroot"], + "cmds_each": ["dnf", "rpm", "yum"], + "each": ["fakeroot"] } }, + + # Debian notes: + # + # 1. By default in recent Debians, apt(8) runs as an unprivileged user. + # This makes *all* apt operations fail in an unprivileged container + # because it can't drop privileges. There are multiple ways to turn the + # “sandbox” off. As far as I can tell, none are documented, but this one + # at least appears in google searches a lot. + # + # apt also doesn't drop privileges if there is no user _apt; in my + # testing, sometimes this user is present and sometimes not, for reasons + # I don't understand. If not present, you get this warning: + # + # W: No sandbox user '_apt' on the system, can not drop privileges + # + # Configuring apt not to use the sandbox seemed cleaner than deleting + # this user and eliminates the warning. + + { "match": ("/etc/debian_version", r"^(9|10)\."), + "config": { "name": "Debian 9 (Stretch) or 10 (Buster)", + "first": +["echo 'APT::Sandbox::User \"root\";' > /etc/apt/apt.conf.d/no-sandbox", + "apt-get update", # base image ships with no package indexes + "apt-get install -y pseudo"], + "cmds_each": ["apt", "apt-get", "dpkg"], + "each": ["fakeroot"] } } +] + + +## Functions ## + +def config(img): + ch.DEBUG("fakeroot: checking configs: %s" % img) + for c in DEFAULT_CONFIGS: + (path, rx) = c["match"] + path_full = "%s/%s" % (img, path) + ch.DEBUG("fakeroot: checking %s: grep '%s' %s" + % (c["config"]["name"], rx, path)) + if (os.path.isfile(path_full) and ch.grep_p(path_full, rx)): + ch.DEBUG("fakeroot: using config %s" % c["config"]["name"]) + return c["config"] + ch.DEBUG("fakeroot: no config found") + return None + +def inject_each(img, args): + c = config(img) + if (c is None): + return args + # Match on words, not substrings. + for each in c["cmds_each"]: + for arg in args: + if (each in arg.split()): + return c["each"] + args + return args + +def inject_first(img, env): + c = config(img) + if (c is None): + return + if (os.path.exists("%s/ch/fakeroot-first-run")): + ch.DEBUG("fakeroot: already initialized") + return + ch.INFO("fakeroot: initializing for %s" % c["name"]) + for cl in c["first"]: + ch.INFO("fakeroot: $ %s" % cl) + args = ["/bin/sh", "-c", cl] + ch.ch_run_modify(img, args, env) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/packaging/fedora/charliecloud.spec new/charliecloud-0.20/packaging/fedora/charliecloud.spec --- old/charliecloud-0.19/packaging/fedora/charliecloud.spec 2020-09-21 22:07:10.000000000 +0200 +++ new/charliecloud-0.20/packaging/fedora/charliecloud.spec 2020-10-10 00:44:26.000000000 +0200 @@ -122,6 +122,7 @@ %{_libdir}/%{name}/build.py %{_libdir}/%{name}/charliecloud.py %{_libdir}/%{name}/contributors.bash +%{_libdir}/%{name}/fakeroot.py %{_libdir}/%{name}/misc.py %{_libdir}/%{name}/version.py %{_libdir}/%{name}/version.sh diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/packaging/vagrant/Vagrantfile new/charliecloud-0.20/packaging/vagrant/Vagrantfile --- old/charliecloud-0.19/packaging/vagrant/Vagrantfile 2020-09-18 17:12:53.000000000 +0200 +++ new/charliecloud-0.20/packaging/vagrant/Vagrantfile 2020-10-20 18:44:46.000000000 +0200 @@ -69,16 +69,16 @@ set -e cd /tmp - # Basic stuff from standard repos. + # Basic stuff from standard repos and confgure yum to fail on missing pkgs. yum makecache fast - yum-config-manager --setopt=deltarpm=0 --save + yum-config-manager --setopt=deltarpm=0 --setopt=skip_missing_names_on_install=0 --save yum -y upgrade yum -y install emacs \ vim \ wget # Git from IUS. This also activates EPEL. - # From here: https://ius.io/setup + # From here: https://ius.io/setup # Likely to be deprecated again sometime in the future wget https://repo.ius.io/ius-release-el7.rpm yum -y install epel-release @@ -254,8 +254,8 @@ echo '%vagrant ALL=(ALL:ALL) NOPASSWD: ALL' > /etc/sudoers.d/vagrant # Configure subuids and subgids for runc. - sudo usermod --add-subuids 10000-65536 vagrant - sudo usermod --add-subgids 10000-65536 vagrant + usermod --add-subuids 10000-65536 vagrant + usermod --add-subgids 10000-65536 vagrant EOF # Remove unneeded packages. @@ -284,8 +284,8 @@ chown -R charlie:charlie /usr/local/src/charliecloud # Configure subuids and subgids for runc. - sudo usermod --add-subuids 10000-65536 charlie - sudo usermod --add-subgids 10000-65536 charlie + usermod --add-subuids 10000-65536 charlie + usermod --add-subgids 10000-65536 charlie # Automatically log in "charlie" on the console, so they have a way to get # in if SSH isn't working. @@ -294,6 +294,10 @@ cp /lib/systemd/system/[email protected] [email protected] sed -ri 's|^ExecStart=.*$|ExecStart=-/sbin/agetty --autologin charlie --noclear %I|' [email protected] + # A test tries to signal a getty process from within the container, so + # we need at least one enabled. + sudo systemctl enable [email protected] + # Configure SSH to allow password logins. We would prefer to keep the # Vagrant default of SSH keys only, but I can't figure out how to get the # key into the VM in a way that's easy for end users. @@ -302,7 +306,7 @@ # Fix /etc/shadow permissions. Not clear where they were broken, but # passwd(1) below fails without this. - sudo restorecon -v /etc/shadow + restorecon -v /etc/shadow # Lock out password login for root and vagrant, because the default # password is well-known and we now allow password login. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/charliecloud-0.19/test/run/ch-run_uidgid.bats new/charliecloud-0.20/test/run/ch-run_uidgid.bats --- old/charliecloud-0.19/test/run/ch-run_uidgid.bats 2020-09-18 17:12:53.000000000 +0200 +++ new/charliecloud-0.20/test/run/ch-run_uidgid.bats 2020-09-24 00:09:29.000000000 +0200 @@ -137,7 +137,12 @@ } @test 'signal process outside container' { - # Send a signal to a process we shouldn't be able to signal. + # Send a signal to a process we shouldn't be able to signal, in this case + # getty. This requires at least one getty running, i.e., at least one + # virtual console waiting for login. In the past, distributions ran gettys + # on several VCs by default, but in recent years they are often started + # dynamically, so there may be none running. See your distro's + # documentation on how to configure this. See also e.g. issue #840. [[ $(pgrep -c getty) -eq 0 ]] && pedantic_fail 'no getty process found' ch-run $uid_args $gid_args "$ch_timg" -- /test/signal_out.py }
