Package: autopkgtest
Version: 4.1
Severity: wishlist
Tags: patch
A project I'm currently working on uses a read-only root filesystem
under normal circumstances, mounting it rw only when an upgrade is
desired. This makes autopkgtest fail horribly when testing our disk
images, which is (at least mostly) addressed by the patches I've
attached here. Please consider.
Thanks,
S
-- System Information:
Debian Release: stretch/sid
APT prefers unstable-debug
APT policy: (500, 'unstable-debug'), (500, 'unstable'), (500, 'testing'),
(500, 'stable'), (1, 'experimental-debug'), (1, 'experimental')
Architecture: amd64 (x86_64)
Foreign Architectures: i386
Kernel: Linux 4.7.0-1-amd64 (SMP w/4 CPU cores)
Locale: LANG=en_GB.utf8, LC_CTYPE=en_GB.utf8 (charmap=UTF-8)
Shell: /bin/sh linked to /bin/dash
Init: systemd (via /run/systemd/system)
Versions of packages autopkgtest depends on:
ii apt-utils 1.3.1
ii libdpkg-perl 1.18.10
ii procps 2:3.3.12-2
ii python3 3.5.1-4
ii python3-debian 0.1.29
Versions of packages autopkgtest recommends:
ii autodep8 0.8
Versions of packages autopkgtest suggests:
pn lxc <none>
pn lxd-client <none>
ii qemu-system 1:2.7+dfsg-1
ii qemu-utils 1:2.7+dfsg-1
ii schroot 1.6.10-2+b1
-- no debconf information
>From 686bc470ce39630c37fd7e3436ba3bc706a8f040 Mon Sep 17 00:00:00 2001
From: Simon McVittie <[email protected]>
Date: Tue, 25 Oct 2016 19:34:24 +0100
Subject: [PATCH 1/6] VirtSubproc: open arbitrary files in binary mode
It doesn't actually matter when we only use their fileno(), but it's
better to be consistent.
Signed-off-by: Simon McVittie <[email protected]>
---
lib/VirtSubproc.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/lib/VirtSubproc.py b/lib/VirtSubproc.py
index ba87740..1aeb673 100644
--- a/lib/VirtSubproc.py
+++ b/lib/VirtSubproc.py
@@ -38,7 +38,7 @@ import shutil
import adtlog
progname = "<VirtSubproc>"
-devnull_read = open('/dev/null', 'r')
+devnull_read = open('/dev/null', 'rb')
caller = __main__
copy_timeout = int(os.getenv('AUTOPKGTEST_VIRT_COPY_TIMEOUT', '300'))
@@ -512,9 +512,9 @@ def copyupdown_internal(wh, sd, upp):
if not dirsp:
rune = 'cat %s%s' % ('><'[upp], remfileq)
if upp:
- deststdout = open(sd[idst], 'w')
+ deststdout = open(sd[idst], 'wb')
else:
- srcstdin = open(sd[isrc], 'r')
+ srcstdin = open(sd[isrc], 'rb')
status = os.fstat(srcstdin.fileno())
if status.st_mode & 0o111:
rune += '; chmod +x -- %s' % (remfileq)
--
2.10.1
>From e46a2dff9d4c93a3d5a99849b3c741e000076084 Mon Sep 17 00:00:00 2001
From: Simon McVittie <[email protected]>
Date: Tue, 25 Oct 2016 19:35:03 +0100
Subject: [PATCH 2/6] VirtSubproc: if check_exec status is nonzero, include
stderr in message
If a command fails, it's usually interesting to know why.
Signed-off-by: Simon McVittie <[email protected]>
---
lib/VirtSubproc.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/lib/VirtSubproc.py b/lib/VirtSubproc.py
index 1aeb673..a20c337 100644
--- a/lib/VirtSubproc.py
+++ b/lib/VirtSubproc.py
@@ -183,8 +183,8 @@ def check_exec(argv, downp=False, outp=False, timeout=0):
stdout=stdout, stderr=subprocess.PIPE)
if status:
- bomb("%s%s failed (exit status %d)" %
- ((downp and "(down) " or ""), argv, status))
+ bomb("%s%s failed (exit status %d, stderr %r)" %
+ ((downp and "(down) " or ""), argv, status, err))
if err:
bomb("%s unexpectedly produced stderr output `%s'" %
(argv, err))
--
2.10.1
>From 815b8fdcceb9259ef186a35698cc41d25cf7ff56 Mon Sep 17 00:00:00 2001
From: Simon McVittie <[email protected]>
Date: Tue, 25 Oct 2016 19:38:02 +0100
Subject: [PATCH 3/6] qemu: put the shared directory in /run
If the virtual machine's root filesystem is read-only, we won't
be able to create /autopkgtest.
Signed-off-by: Simon McVittie <[email protected]>
---
virt/autopkgtest-virt-qemu | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/virt/autopkgtest-virt-qemu b/virt/autopkgtest-virt-qemu
index cdae1de..d9c59a2 100755
--- a/virt/autopkgtest-virt-qemu
+++ b/virt/autopkgtest-virt-qemu
@@ -218,10 +218,10 @@ def setup_shared(shared_dir):
term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
- term.send(b'''mkdir -p -m 1777 /autopkgtest
-mount -t 9p -o trans=virtio,access=any autopkgtest /autopkgtest
-chmod 1777 /autopkgtest
-touch /autopkgtest/done_shared
+ term.send(b'''mkdir -p -m 1777 /run/autopkgtest/shared
+mount -t 9p -o trans=virtio,access=any autopkgtest /run/autopkgtest/shared
+chmod 1777 /run/autopkgtest/shared
+touch /run/autopkgtest/shared/done_shared
''')
with VirtSubproc.timeout(10, 'timed out on client shared directory setup'):
@@ -273,8 +273,8 @@ def setup_config(shared_dir):
# eternally (like tail -f), but stop once either an "EOF" file exists and
# we copied at least as many bytes as given in that EOF file (the first
# arg), or an "exit flag" file exists.
- # We don't run that from /autopkgtest/ as 9p from older QEMU versions is
- # buggy and causes "invalid numeric result" errors on that.
+ # We don't run that from /run/autopkgtest/shared as 9p from older QEMU
+ # versions is buggy and causes "invalid numeric result" errors on that.
term.send(b'''PYTHON=$(which python3) || PYTHON=$(which python); cat <<EOF > /bin/eofcat; chmod 755 /bin/eofcat
#!$PYTHON
import sys, os, fcntl, time, errno
@@ -321,7 +321,7 @@ dir_host = '%(dir)s'
job_host = tempfile.mkdtemp(prefix='job.', dir=dir_host)
atexit.register(shutil.rmtree, job_host)
os.chmod(job_host, 0o755)
-job_guest = '/autopkgtest/' + os.path.basename(job_host)
+job_guest = '/run/autopkgtest/shared/' + os.path.basename(job_host)
running = True
def shovel(fin, fout, flagfile_on_eof=None):
@@ -495,7 +495,7 @@ def determine_normal_user(shared_dir):
term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
term.send(b"getent passwd | sort -t: -nk3 | "
b"awk -F: '{if ($3 >= 500) { print $1; exit } }'"
- b"> /autopkgtest/normal_user\n")
+ b"> /run/autopkgtest/shared/normal_user\n")
with VirtSubproc.timeout(5, 'timed out on determining normal user'):
outfile = os.path.join(shared_dir, 'normal_user')
while not os.path.exists(outfile):
@@ -580,7 +580,7 @@ def hook_open():
def hook_downtmp(path):
# we would like to do this, but 9p is currently way too slow for big source
# trees
- # downtmp = '/autopkgtest/tmp'
+ # downtmp = '/run/autopkgtest/shared/tmp'
# VirtSubproc.check_exec(['mkdir', '-m', '1777', downtmp], downp=True)
return VirtSubproc.downtmp_mktemp(path)
--
2.10.1
>From 5c31b67c43f2d6c638b6ac7f2b2a8cfffe1d0c11 Mon Sep 17 00:00:00 2001
From: Simon McVittie <[email protected]>
Date: Tue, 25 Oct 2016 19:43:32 +0100
Subject: [PATCH 4/6] virt-qemu: re-create eofcat on every boot
This is currently useless, but will become useful when I put it
on a tmpfs.
Signed-off-by: Simon McVittie <[email protected]>
---
virt/autopkgtest-virt-qemu | 70 +++++++++++++++++++++++-----------------------
1 file changed, 35 insertions(+), 35 deletions(-)
diff --git a/virt/autopkgtest-virt-qemu b/virt/autopkgtest-virt-qemu
index d9c59a2..24e4353 100755
--- a/virt/autopkgtest-virt-qemu
+++ b/virt/autopkgtest-virt-qemu
@@ -234,41 +234,6 @@ touch /run/autopkgtest/shared/done_shared
term.send(b'[ -n "$HOME" ] || export HOME=`getent passwd root|cut -f6 -d:`\n')
VirtSubproc.expect(term, b'#', 5)
-
-def setup_config(shared_dir):
- '''Set up configuration files'''
-
- term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
-
- # copy our timezone, to avoid time skews with the host
- if os.path.exists('/etc/timezone'):
- tz = None
- with open('/etc/timezone', 'rb') as f:
- for l in f:
- if l.startswith(b'#'):
- continue
- l = l.strip()
- if l:
- tz = l
- break
-
- if tz:
- adtlog.debug('Copying host timezone %s to VM' % tz.decode())
- term.send(b'echo ' + tz + b' > /etc/timezone; DEBIAN_FRONTEND=noninteractive dpkg-reconfigure tzdata\n')
- VirtSubproc.expect(term, b'#', 30)
- else:
- adtlog.debug('Could not determine host timezone')
-
- # ensure that we have Python for our the auxverb helpers
- term.send(b'type python3 2>/dev/null || type python 2>/dev/null\n')
- try:
- out = VirtSubproc.expect(term, b'/python', 5)
- except VirtSubproc.Timeout:
- VirtSubproc.bomb('Neither python3 nor python is installed in the VM, '
- 'one of them is required by autopkgtest')
- if b'\n# ' not in out:
- VirtSubproc.expect(term, b'# ', 5)
-
# create helper for runcmd: cat data from its stdin (from a file) to stdout
# eternally (like tail -f), but stop once either an "EOF" file exists and
# we copied at least as many bytes as given in that EOF file (the first
@@ -308,6 +273,41 @@ EOF
VirtSubproc.expect(term, b'# ', 5)
+def setup_config(shared_dir):
+ '''Set up configuration files'''
+
+ term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
+
+ # copy our timezone, to avoid time skews with the host
+ if os.path.exists('/etc/timezone'):
+ tz = None
+ with open('/etc/timezone', 'rb') as f:
+ for l in f:
+ if l.startswith(b'#'):
+ continue
+ l = l.strip()
+ if l:
+ tz = l
+ break
+
+ if tz:
+ adtlog.debug('Copying host timezone %s to VM' % tz.decode())
+ term.send(b'echo ' + tz + b' > /etc/timezone; DEBIAN_FRONTEND=noninteractive dpkg-reconfigure tzdata\n')
+ VirtSubproc.expect(term, b'#', 30)
+ else:
+ adtlog.debug('Could not determine host timezone')
+
+ # ensure that we have Python for our the auxverb helpers
+ term.send(b'type python3 2>/dev/null || type python 2>/dev/null\n')
+ try:
+ out = VirtSubproc.expect(term, b'/python', 5)
+ except VirtSubproc.Timeout:
+ VirtSubproc.bomb('Neither python3 nor python is installed in the VM, '
+ 'one of them is required by autopkgtest')
+ if b'\n# ' not in out:
+ VirtSubproc.expect(term, b'# ', 5)
+
+
def make_auxverb(shared_dir):
'''Create auxverb script'''
--
2.10.1
>From d546a68744d2f98caac8db7ea08eb4c71f23a7cd Mon Sep 17 00:00:00 2001
From: Simon McVittie <[email protected]>
Date: Tue, 25 Oct 2016 19:42:07 +0100
Subject: [PATCH 5/6] qemu: put eofcat in an executable tmpfs on
/run/autopkgtest
This avoids having to write it to /bin, which might be read-only.
Signed-off-by: Simon McVittie <[email protected]>
---
virt/autopkgtest-virt-qemu | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/virt/autopkgtest-virt-qemu b/virt/autopkgtest-virt-qemu
index 24e4353..98f0317 100755
--- a/virt/autopkgtest-virt-qemu
+++ b/virt/autopkgtest-virt-qemu
@@ -218,7 +218,9 @@ def setup_shared(shared_dir):
term = VirtSubproc.get_unix_socket(os.path.join(workdir, 'ttyS1'))
- term.send(b'''mkdir -p -m 1777 /run/autopkgtest/shared
+ term.send(b'''mkdir -p /run/autopkgtest
+mount -t tmpfs run-autopkgtest /run/autopkgtest
+mkdir -p -m 1777 /run/autopkgtest/shared
mount -t 9p -o trans=virtio,access=any autopkgtest /run/autopkgtest/shared
chmod 1777 /run/autopkgtest/shared
touch /run/autopkgtest/shared/done_shared
@@ -240,7 +242,7 @@ touch /run/autopkgtest/shared/done_shared
# arg), or an "exit flag" file exists.
# We don't run that from /run/autopkgtest/shared as 9p from older QEMU
# versions is buggy and causes "invalid numeric result" errors on that.
- term.send(b'''PYTHON=$(which python3) || PYTHON=$(which python); cat <<EOF > /bin/eofcat; chmod 755 /bin/eofcat
+ term.send(b'''PYTHON=$(which python3) || PYTHON=$(which python); cat <<EOF > /run/autopkgtest/eofcat; chmod 755 /run/autopkgtest/eofcat
#!$PYTHON
import sys, os, fcntl, time, errno
(feof, fexit) = sys.argv[1:]
@@ -376,7 +378,7 @@ t_stderr.start()
# "real" stdin behaviour.
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
s.connect('%(tty)s')
-cmd = 'PYTHONHASHSEED=0 /bin/eofcat %%(d)s/stdin_eof %%(d)s/exit.tmp < %%(d)s/stdin | ' \\
+cmd = 'PYTHONHASHSEED=0 /run/autopkgtest/eofcat %%(d)s/stdin_eof %%(d)s/exit.tmp < %%(d)s/stdin | ' \\
'(%%(c)s >> %%(d)s/stdout 2>> %%(d)s/stderr; echo $? > %%(d)s/exit.tmp);' \\
'mv %%(d)s/exit.tmp %%(d)s/exit\\n' %% \\
{'d': job_guest, 'c': ' '.join(map(pipes.quote, sys.argv[1:]))}
--
2.10.1
>From bdb5b13aaa20946ac9ccebffe55e7747f4ad54ec Mon Sep 17 00:00:00 2001
From: Simon McVittie <[email protected]>
Date: Tue, 25 Oct 2016 20:01:10 +0100
Subject: [PATCH 6/6] testbed: mount / rw before invoking dpkg/apt
This assumes we can "apt-get update" without remounting (in other words,
/var is always rw), but might need to remount the root read/write
before proceeding.
Signed-off-by: Simon McVittie <[email protected]>
---
lib/adt_binaries.py | 1 +
lib/adt_run_args.py | 1 +
lib/adt_testbed.py | 1 +
3 files changed, 3 insertions(+)
diff --git a/lib/adt_binaries.py b/lib/adt_binaries.py
index c723c37..21c9e13 100644
--- a/lib/adt_binaries.py
+++ b/lib/adt_binaries.py
@@ -125,6 +125,7 @@ class DebBinaries:
adtlog.debug('Binaries: publish reinstall needs ' + pkg)
if pkgs_reinstall:
+ self.testbed.check_exec(['mount', '-o', 'remount,rw', '/'])
rc = self.testbed.execute(
['apt-get', '--quiet', '-o', 'Debug::pkgProblemResolver=true',
'-o', 'APT::Get::force-yes=true',
diff --git a/lib/adt_run_args.py b/lib/adt_run_args.py
index 7f6aa3e..2e50562 100644
--- a/lib/adt_run_args.py
+++ b/lib/adt_run_args.py
@@ -267,6 +267,7 @@ details.'''
action='append_const',
const='(apt-get update || (sleep 15; apt-get update)'
' || (sleep 60; apt-get update) || false)'
+ ' && mount -o remount,rw /'
' && $(which eatmydata || true) apt-get dist-upgrade -y -o '
'Dpkg::Options::="--force-confnew"',
help='Run apt update/dist-upgrade before the tests')
diff --git a/lib/adt_testbed.py b/lib/adt_testbed.py
index 7fc1158..a0b6a67 100644
--- a/lib/adt_testbed.py
+++ b/lib/adt_testbed.py
@@ -499,6 +499,7 @@ Description: satisfy autopkgtest test dependencies
# so we might need to retry without pinning
download_fail_retries = 3
while True:
+ self.check_exec(['mount', '-o', 'remount,rw', '/'])
self.check_exec(['dpkg', '--unpack', deb.tb], stdout=subprocess.PIPE)
# capture status-fd to stderr
(rc, _, serr) = self.execute(['/bin/sh', '-ec', '%s apt-get install '
--
2.10.1
