Faidon Liambotis has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/338383 )
Change subject: mirrors: update archvsync to 20170204
......................................................................
mirrors: update archvsync to 20170204
Update from 20160306 to 20170204. This versions moves etc/common to
bin/common (finally!) and allows the config files to be installed under
.config/archvsync or /etc/archvsync. Prepares the ground for switching
to real FHS paths and possibly the package, but don't do that just yet.
Change-Id: Id40c4132d010004df74fe26b542d2def9718b74e
---
R modules/mirrors/files/archvsync/bin/common
M modules/mirrors/files/archvsync/bin/ftpsync
M modules/mirrors/manifests/debian.pp
3 files changed, 50 insertions(+), 25 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/83/338383/1
diff --git a/modules/mirrors/files/archvsync/etc/common
b/modules/mirrors/files/archvsync/bin/common
similarity index 94%
rename from modules/mirrors/files/archvsync/etc/common
rename to modules/mirrors/files/archvsync/bin/common
index fd4cebb..f97016e 100644
--- a/modules/mirrors/files/archvsync/etc/common
+++ b/modules/mirrors/files/archvsync/bin/common
@@ -261,3 +261,32 @@
fi
unset RSYNC_VERSION RSYNC_REGEX
}
+
+# Search config files in various locations
+search_config() {
+ local file
+ if [ "$BASEDIR" ]; then
+ file="$BASEDIR/etc/$1"
+ if [ -f "$file" ]; then
+ echo "$file"
+ return
+ fi
+ fi
+ for i in ~/.config/archvsync /etc/archvsync; do
+ file="$i/$1"
+ if [ -f "$file" ]; then
+ echo "$file"
+ return
+ fi
+ done
+}
+
+# Read config file
+read_config() {
+ local config=$(search_config "$1")
+ if [ "$config" ]; then
+ . "$config"
+ return 0
+ fi
+ return 1
+}
diff --git a/modules/mirrors/files/archvsync/bin/ftpsync
b/modules/mirrors/files/archvsync/bin/ftpsync
index 931fe09..e3960fa 100755
--- a/modules/mirrors/files/archvsync/bin/ftpsync
+++ b/modules/mirrors/files/archvsync/bin/ftpsync
@@ -42,10 +42,10 @@
# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
# by Joerg Jaspert and the Debian mirroradm group.
# This is used to track which mirror is using which script version.
-VERSION="20160306"
+VERSION="20170204"
# Source our common functions
-. "${BASEDIR}/etc/common"
+. "${BASEDIR}/bin/common"
########################################################################
########################################################################
@@ -200,24 +200,15 @@
echo "Used ftpsync version: ${VERSION}"
echo "Running on host: ${TRACEHOST}"
if [[ full = ${EXTENDEDTRACE} ]]; then
- ARCHLIST=""
+ ARCH=$((find ${TO}/dists -name Release -exec sed
-ne 's/^Architecture: //p' '{}' + 2>/dev/null || echo "unknown") | sort -u | tr
'\n' ' ')
+ echo "Architectures: ${ARCH}"
if [[ ${ARCH_INCLUDE} ]]; then
- AINCLUDE="^${ARCH_INCLUDE// /\$|^}$"
- for ARCH in ${GLOBALARCHLIST}; do
- if [[ ${ARCH} =~ ${AINCLUDE} ]]; then
- ARCHLIST="${ARCHLIST} ${ARCH}"
- fi
- done
+ echo "Architectures-Configuration: INCLUDE
$(tr ' ' '\n' <<< ${ARCH_INCLUDE} | sort -u | tr '\n' ' ')"
+ elif [[ ${ARCH_EXCLUDE} ]]; then
+ echo "Architectures-Configuration: EXCLUDE
$(tr ' ' '\n' <<< ${ARCH_EXCLUDE} | sort -u | tr '\n' ' ')"
else
- AEXCLUDE="^${ARCH_EXCLUDE// /\$|^}$"
- for ARCH in ${GLOBALARCHLIST}; do
- if ! [[ ${ARCH} =~ ${AEXCLUDE} ]]; then
- ARCHLIST="${ARCHLIST} ${ARCH}"
- fi
- done
+ echo "Architectures-Configuration: ALL"
fi
- out="GUESSED:{${ARCHLIST}}"
- echo "Architectures: ${out}"
echo "Upstream-mirror: ${RSYNC_HOST}"
echo "SSL: ${RSYNC_SSL}"
total=0
@@ -364,9 +355,7 @@
# Now source the config for the archive we run on.
# (Yes, people can also overwrite the options above in the config file
# if they want to)
-if [[ -f ${BASEDIR}/etc/${NAME}.conf ]]; then
- . "${BASEDIR}/etc/${NAME}.conf"
-else
+if ! read_config "${NAME}.conf"; then
echo "Nono, you can't tell us about random archives. Bad boy!"
exit 1
fi
@@ -452,7 +441,7 @@
# Default rsync options for *every* rsync call
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats
${RSYNC_FILTER}"}
# Options we only use in the first pass, where we do not want packages/sources
to fly in yet and don't want to delete files
-RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources*
--exclude=Release* --exclude=InRelease --exclude=i18n/* --exclude=ls-lR*"}
+RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources*
--exclude=Release* --exclude=InRelease --include=i18n/by-hash/**
--exclude=i18n/* --exclude=ls-lR*"}
# Options for the second pass, where we do want everything, including deletion
of old and now unused files
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete
--delete-excluded"}
# Which rsync share to use on our upstream mirror?
@@ -546,8 +535,6 @@
# Really nothing to see below here. Only code follows. #
########################################################################
########################################################################
-GLOBALARCHLIST="source amd64 arm64 armel armhf hurd-i386 i386 ia64
kfreebsd-amd64 kfreebsd-i386 mips mips64el mipsel powerpc ppc64el s390 s390x
sparc"
-
DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R)
# Exclude architectures defined in $ARCH_EXCLUDE
@@ -606,6 +593,7 @@
trap cleanup ERR TERM HUP INT QUIT
# Start log by redirecting stdout and stderr there and closing stdin
+mkdir -p "$LOGDIR"
exec >"$LOG" 2>&1 <&-
log "Mirrorsync start"
diff --git a/modules/mirrors/manifests/debian.pp
b/modules/mirrors/manifests/debian.pp
index f98d1f5..3b72275 100644
--- a/modules/mirrors/manifests/debian.pp
+++ b/modules/mirrors/manifests/debian.pp
@@ -24,8 +24,8 @@
}
# this is <https://ftp-master.debian.org/git/archvsync.git>
- # right now we just ship bin/ftpsync & etc/common; if more are needed in
- # the future this should probably become a git::install resource
+ # right now we just ship bin/ftpsync & bin/common
+ # there is soon going to be a Debian package, use that then instead
file { "${mirrors::homedir}/archvsync":
ensure => directory,
recurse => true,
@@ -45,6 +45,14 @@
mode => '0755',
}
+ # move to .config/archvsync or /etc/archvsync?
+ file { "${mirrors::homedir}/archvsync/etc":
+ ensure => directory,
+ owner => 'mirror',
+ group => 'mirror',
+ mode => '0755',
+ }
+
# this is our configuration for archvsync
file { "${mirrors::homedir}/archvsync/etc/ftpsync.conf":
ensure => present,
--
To view, visit https://gerrit.wikimedia.org/r/338383
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id40c4132d010004df74fe26b542d2def9718b74e
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Faidon Liambotis <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
