commit: 151b13ac968bc3eab8226e20489f193b8b3b48d7
Author: Ben Kohler <bkohler <AT> gentoo <DOT> org>
AuthorDate: Fri May 15 13:02:33 2020 +0000
Commit: Ben Kohler <bkohler <AT> gentoo <DOT> org>
CommitDate: Fri May 15 13:02:33 2020 +0000
URL: https://gitweb.gentoo.org/proj/releng.git/commit/?id=151b13ac
catalyst-auto-x86*.conf: explicitly list files to be transferred
Previously when we were copying locally there was no (major) problem
with wildcarding a long list of files to be transferred after every
single spec finishes. Now that we are transferring to a remote host,
this results in a lot of wasted transfer time & bandwidth.
I have adjusted the post_build function to only transfer the new files
known to be produced by each spec (which has files meant to be
published).
Signed-off-by: Ben Kohler <bkohler <AT> gentoo.org>
tools/catalyst-auto-x86-experimental.conf | 71 +++++++++++++++++--------------
tools/catalyst-auto-x86.conf | 57 ++++++++++++-------------
2 files changed, 67 insertions(+), 61 deletions(-)
diff --git a/tools/catalyst-auto-x86-experimental.conf
b/tools/catalyst-auto-x86-experimental.conf
index 3fd60287..061cab92 100644
--- a/tools/catalyst-auto-x86-experimental.conf
+++ b/tools/catalyst-auto-x86-experimental.conf
@@ -30,17 +30,27 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
update_symlinks() {
# Symlink the latest stages3 to build from
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
- pushd $d >/dev/null
+ pushd "${d}" >/dev/null || exit
for f in $(ls stage3*${EXTENSIONS} | grep -v latest |
give_latest_from_dates ) ; do
- of=${f/20[0-9][0-9].[0-9]/latest} # for 20XX.Y stuff
- of=${of/20[0-9][0-9][0-1][0-9][0-9][0-9]/latest} # for
20YYMMDD stuff
- ln -sf $f $of
+ # 20yymmddThhmmssZ
+ # 20yymmddhhmmss
+ # 20yymmdd
+ # 20yy.n
+ of=$(perl -p \
+ -e 's/20\d{6}T\d{6}Z/latest/g;' \
+ -e 's/20\d{6}\d{6}/latest/g;' \
+ -e 's/20\d{6}/latest/g;' \
+ -e 's/20\d{2}\.\d{2}/latest/g;' \
+ <<<"$f")
+ ln -svf "$f" "$of"
done
- popd >/dev/null
+ popd >/dev/null || exit
done
}
post_build() {
+ local set=$1 spec=$2
+
cmd=(
rsync
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o
VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -59,34 +69,31 @@ post_build() {
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
;;
esac
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
- mkdir -p ${TMPDIR}/empty
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
- for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS}
); do
- if [ -f $file ]; then
- "${cmd[@]}" ${file}* ${DEST_DEFAULT}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
- fi
- popd >/dev/null
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
- for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS}
); do
- if [ -f $file ]; then
- "${cmd[@]}" $file* ${DEST_HARDENED}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
- fi
- popd >/dev/null
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+ case ${spec} in
+ stage3.spec)
+ "${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ installcd-stage2-minimal.spec)
+ "${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso*
"${DEST_DEFAULT}"
+ ;;
+ i686/stage3.spec)
+ "${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ esac
+ popd >/dev/null || exit
+
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+ case ${spec} in
+ hardened/stage3.spec)
+ "${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz*
"${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2.spec)
+ "${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ esac
+ popd >/dev/null || exit
}
# vim:ft=sh:
diff --git a/tools/catalyst-auto-x86.conf b/tools/catalyst-auto-x86.conf
index 27637e59..e778ce8e 100644
--- a/tools/catalyst-auto-x86.conf
+++ b/tools/catalyst-auto-x86.conf
@@ -30,7 +30,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
update_symlinks() {
# Symlink the latest stages3 to build from
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
- pushd $d >/dev/null
+ pushd "${d}" >/dev/null || exit
for f in $(ls stage3*${EXTENSIONS} | grep -v latest |
give_latest_from_dates ) ; do
# 20yymmddThhmmssZ
# 20yymmddhhmmss
@@ -44,11 +44,13 @@ update_symlinks() {
<<<"$f")
ln -svf "$f" "$of"
done
- popd >/dev/null
+ popd >/dev/null || exit
done
}
post_build() {
+ local set=$1 spec=$2
+
cmd=(
rsync
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o
VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -67,34 +69,31 @@ post_build() {
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
;;
esac
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
- mkdir -p ${TMPDIR}/empty
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS}
stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" ${file}* ${DEST_DEFAULT}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
- fi
- popd >/dev/null
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS}
stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
- if [ -f $file ]; then
- "${cmd[@]}" $file* ${DEST_HARDENED}
- fi
- done
- if [ -f *${DATESTAMP}*.iso ]; then
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
- elif [ -f *${TIMESTAMP}*.iso ]; then
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
- fi
- popd >/dev/null
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+ case ${spec} in
+ stage3.spec)
+ "${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ installcd-stage2-minimal.spec)
+ "${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso*
"${DEST_DEFAULT}"
+ ;;
+ i686/stage3.spec)
+ "${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+ ;;
+ esac
+ popd >/dev/null || exit
+
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+ case ${spec} in
+ hardened/stage3.spec)
+ "${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz*
"${DEST_HARDENED}"
+ ;;
+ hardened/admincd-stage2.spec)
+ "${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+ ;;
+ esac
+ popd >/dev/null || exit
}
# vim:ft=sh:
