Faidon Liambotis has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/384791 )
Change subject: mirrors: stop shipping ftpsync, use package instead
......................................................................
mirrors: stop shipping ftpsync, use package instead
ftpsync (src:archvsync) is now available as a package, albeit only in
buster (and now, stretch-wikimedia). Switch to using that instead of
shipping a copy in our puppet repository, thereby making updates more
streamlined, especially in >= buster.
Change-Id: I1cca0f806a014b5c05f7296b78eda7c2004c79a9
---
D modules/mirrors/files/archvsync/bin/common
D modules/mirrors/files/archvsync/bin/ftpsync
M modules/mirrors/files/ssh-debian-archvsync.pub
M modules/mirrors/manifests/debian.pp
M modules/mirrors/spec/classes/mirrors_debian_spec.rb
M modules/mirrors/templates/ftpsync.conf.erb
6 files changed, 19 insertions(+), 1,191 deletions(-)
Approvals:
Faidon Liambotis: Looks good to me, approved
jenkins-bot: Verified
diff --git a/modules/mirrors/files/archvsync/bin/common
b/modules/mirrors/files/archvsync/bin/common
deleted file mode 100644
index f97016e..0000000
--- a/modules/mirrors/files/archvsync/bin/common
+++ /dev/null
@@ -1,292 +0,0 @@
-# -*- mode:sh -*-
-# vim:syn=sh
-# Little common functions
-
-# push a mirror attached to us.
-# Arguments (using an array named SIGNAL_OPTS):
-#
-# $MIRROR - Name for the mirror, also basename for the logfile
-# $HOSTNAME - Hostname to push to
-# $USERNAME - Username there
-# $SSHPROTO - Protocol version, either 1 or 2.
-# $SSHKEY - the ssh private key file to use for this push
-# $SSHOPTS - any other option ssh accepts, passed blindly, be careful
-# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged
-# $PUSHTYPE - what kind of push should be done?
-# all - normal, just push once with ssh backgrounded and
finish
-# staged - staged. first push stage1, then wait for $PUSHLOCKs
to appear,
-# then push stage2
-# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!)
-# $PUSHCB - do we want a callback?
-# $PUSHKIND - whats going on? are we doing mhop push or already stage2?
-# $FROMFTPSYNC - set to true if we run from within ftpsync.
-#
-# This function assumes that the variable LOG is set to a directory where
-# logfiles can be written to.
-# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings
-# (list of "lock"files) to wait for if you want pushtype=staged
-#
-# Pushes might be done in background (for type all).
-signal () {
- ARGS="SIGNAL_OPTS[*]"
- local ${!ARGS}
-
- MIRROR=${MIRROR:-""}
- HOSTNAME=${HOSTNAME:-""}
- USERNAME=${USERNAME:-""}
- SSHPROTO=${SSHPROTO:-""}
- SSHKEY=${SSHKEY:-""}
- SSHOPTS=${SSHOPTS:-""}
- PUSHLOCKOWN=${PUSHLOCKOWN:-""}
- PUSHTYPE=${PUSHTYPE:-"all"}
- PUSHARCHIVE=${PUSHARCHIVE:-""}
- PUSHCB=${PUSHCB:-""}
- PUSHKIND=${PUSHKIND:-"all"}
- FROMFTPSYNC=${FROMFTPSYNC:-"false"}
-
- # And now get # back to space...
- SSHOPTS=${SSHOPTS/\#/ }
-
- # Defaults we always want, no matter what
- SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o
ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
-
- # If there are userdefined ssh options, add them.
- if [[ -n ${SSH_OPTS} ]]; then
- SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}"
- fi
-
- # Does this machine need a special key?
- if [[ -n ${SSHKEY} ]]; then
- SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}"
- fi
-
- # Does this machine have an extra own set of ssh options?
- if [[ -n ${SSHOPTS} ]]; then
- SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}"
- fi
-
- # Set the protocol version
- if [[ ${SSHPROTO} -ne 1 ]] && [[ ${SSHPROTO} -ne 2 ]] && [[ ${SSHPROTO}
-ne 99 ]]; then
- # Idiots, we only want 1 or 2. Cant decide? Lets force 2.
- SSHPROTO=2
- fi
-
- if [[ -n ${SSHPROTO} ]] && [[ ${SSHPROTO} -ne 99 ]]; then
- SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}"
- fi
-
- date -u >> "${LOGDIR}/${MIRROR}.log"
-
- PUSHARGS=""
- # PUSHARCHIVE empty or not, we always add the sync:archive: command to
transfer.
- # Otherwise, if nothing else is added, ssh -f would not work ("no command
to execute")
- # But ftpsync does treat "sync:archive:" as the main archive, so this
works nicely.
- PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}"
-
- # We have a callback wish, tell downstreams
- if [[ -n ${PUSHCB} ]]; then
- PUSHARGS="${PUSHARGS} sync:callback"
- fi
- # If we are running an mhop push AND our downstream is one to receive it,
tell it.
- if [[ mhop = ${PUSHKIND} ]] && [[ mhop = ${PUSHTYPE} ]]; then
- PUSHARGS="${PUSHARGS} sync:mhop"
- fi
-
- if [[ all = ${PUSHTYPE} ]]; then
- # Default normal "fire and forget" push. We background that, we do not
care about the mirrors doings
- log "Sending normal push" >> "${LOGDIR}/${MIRROR}.log"
- PUSHARGS1="sync:all"
- ssh -n $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}"
>>"${LOGDIR}/${MIRROR}.log" 2>&1
- if [[ $? -eq 255 ]]; then
- error "Trigger to ${HOSTNAME} failed" >> "${LOG}"
- else
- log "Trigger to ${HOSTNAME} succeed" >> "${LOG}"
- fi
- elif [[ staged = ${PUSHTYPE} ]] || [[ mhop = ${PUSHTYPE} ]]; then
- # Want a staged push. Fine, lets do that. Not backgrounded. We care
about the mirrors doings.
- log "Sending staged push" >> "${LOGDIR}/${MIRROR}.log"
-
- # Only send stage1 if we havent already send it. When called with
stage2, we already did.
- if [[ stage2 != ${PUSHKIND} ]]; then
- # Step1: Do a push to only sync stage1, do not background
- PUSHARGS1="sync:stage1"
- ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}"
>>"${LOGDIR}/${MIRROR}.log" 2>&1
- if [[ $? -eq 255 ]]; then
- error "Trigger to ${HOSTNAME} failed" >> "${LOG}"
- else
- log "Trigger to ${HOSTNAME} succeed" >> "${LOG}"
- fi
- touch "${PUSHLOCKOWN}"
-
- # Step2: Wait for all the other "lock"files to appear.
- tries=0
- # We do not wait forever
- while [[ ${tries} -lt ${PUSHDELAY} ]]; do
- total=0
- found=0
- for file in ${PUSHLOCKS}; do
- total=$(( total + 1 ))
- if [[ -f ${file} ]]; then
- found=$(( found + 1 ))
- fi
- done
- if [[ ${total} -eq ${found} ]] || [[ -f ${LOCKDIR}/all_stage1
]]; then
- touch "${LOCKDIR}/all_stage1"
- break
- fi
- tries=$(( tries + 5 ))
- sleep 5
- done
- # In case we did not have all PUSHLOCKS and still continued, note
it
- # This is a little racy, especially if the other parts decide to
do this
- # at the same time, but it wont hurt more than a mail too much, so
I don't care much
- if [[ ${tries} -ge ${PUSHDELAY} ]]; then
- log "Failed to wait for all other mirrors. Failed ones are:"
>> "${LOGDIR}/${MIRROR}.log"
- for file in ${PUSHLOCKS}; do
- if [[ ! -f ${file} ]]; then
- log "${file}" >> "${LOGDIR}/${MIRROR}.log"
- error "Missing Pushlockfile ${file} after waiting
${tries} second, continuing"
- fi
- done
- fi
- rm -f "${PUSHLOCKOWN}"
- fi
-
- # Step3: It either timed out or we have all the "lock"files, do the
rest
- # If we are doing mhop AND are called from ftpsync - we now exit.
- # That way we notify our uplink that we and all our clients are done
with their
- # stage1. It can then finish its own, and if all our upstreams
downlinks are done,
- # it will send us stage2.
- # If we are not doing mhop or are not called from ftpsync, we start
stage2
- if [[ true = ${FROMFTPSYNC} ]] && [[ mhop = ${PUSHKIND} ]]; then
- return
- else
- PUSHARGS2="sync:stage2"
- log "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log"
- ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}"
>>"${LOGDIR}/${MIRROR}.log" 2>&1
- if [[ $? -eq 255 ]]; then
- error "Trigger to ${HOSTNAME} failed" >> "${LOG}"
- else
- log "Trigger to ${HOSTNAME} succeed" >> "${LOG}"
- fi
- fi
- else
- # Can't decide? Then you get nothing.
- return
- fi
-}
-
-# callback, used by ftpsync
-callback () {
- # Defaults we always want, no matter what
- SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o
ConnectTimeout=45 -o PasswordAuthentication=no"
- ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME}
-}
-
-# log something (basically echo it together with a timestamp)
-#
-# Set $PROGRAM to a string to have it added to the output.
-log () {
- if [[ -z "${PROGRAM}" ]]; then
- echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@"
- else
- echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@"
- fi
-}
-
-# log the message using log() but then also send a mail
-# to the address configured in MAILTO (if non-empty)
-error () {
- log "$@"
- if [[ -n "${MAILTO}" ]]; then
- echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO}
- fi
-}
-
-# run a hook
-# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR
-# the script to run.
-hook () {
- ARGS='HOOK[@]'
- local "${!ARGS}"
- if [[ -n ${HOOKSCR} ]]; then
- log "Running hook $HOOKNR: ${HOOKSCR}"
- set +e
- ${HOOKSCR}
- result=$?
- set -e
- if [[ ${result} -ne 0 ]] ; then
- error "Back from hook $HOOKNR, got returncode ${result}"
- else
- log "Back from hook $HOOKNR, got returncode ${result}"
- fi
- return $result
- else
- return 0
- fi
-}
-
-# Return the list of 2-stage mirrors.
-get2stage() {
- egrep '^(staged|mhop)' "${MIRRORS}" | {
- while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
- PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}"
- done
- echo "$PUSHLOCKS"
- }
-}
-
-# Rotate logfiles
-savelog() {
- torotate="$1"
- count=${2:-${LOGROTATE}}
- while [[ ${count} -gt 0 ]]; do
- prev=$(( count - 1 ))
- if [[ -e ${torotate}.${prev} ]]; then
- mv "${torotate}.${prev}" "${torotate}.${count}"
- fi
- count=$prev
- done
- if [[ -e ${torotate} ]]; then
- mv "${torotate}" "${torotate}.0"
- fi
-}
-
-# Return rsync version
-rsync_protocol() {
- RSYNC_VERSION="$(${RSYNC} --version)"
- RSYNC_REGEX="(protocol[ ]+version[ ]+([0-9]+))"
- if [[ ${RSYNC_VERSION} =~ ${RSYNC_REGEX} ]]; then
- echo ${BASH_REMATCH[2]}
- fi
- unset RSYNC_VERSION RSYNC_REGEX
-}
-
-# Search config files in various locations
-search_config() {
- local file
- if [ "$BASEDIR" ]; then
- file="$BASEDIR/etc/$1"
- if [ -f "$file" ]; then
- echo "$file"
- return
- fi
- fi
- for i in ~/.config/archvsync /etc/archvsync; do
- file="$i/$1"
- if [ -f "$file" ]; then
- echo "$file"
- return
- fi
- done
-}
-
-# Read config file
-read_config() {
- local config=$(search_config "$1")
- if [ "$config" ]; then
- . "$config"
- return 0
- fi
- return 1
-}
diff --git a/modules/mirrors/files/archvsync/bin/ftpsync
b/modules/mirrors/files/archvsync/bin/ftpsync
deleted file mode 100755
index e3960fa..0000000
--- a/modules/mirrors/files/archvsync/bin/ftpsync
+++ /dev/null
@@ -1,857 +0,0 @@
-#! /bin/bash
-# No, we can not deal with sh alone.
-
-set -e
-set -u
-# ERR traps should be inherited from functions too. (And command
-# substitutions and subshells and whatnot, but for us the function is
-# the important part here)
-set -E
-
-# A pipeline's return status is the value of the last (rightmost)
-# command to exit with a non-zero status, or zero if all commands exit
-# success fully.
-set -o pipefail
-
-# ftpsync script for Debian
-# Based losely on a number of existing scripts, written by an
-# unknown number of different people over the years.
-#
-# Copyright (C) 2008-2016 Joerg Jaspert <jo...@debian.org>
-# Copyright (C) 2016 Peter Palfrader
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License as
-# published by the Free Software Foundation; version 2.
-#
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-# In case the admin somehow wants to have this script located someplace else,
-# he can set BASEDIR, and we will take that. If it is unset we use $0
-# How the admin sets this isn't our place to deal with. One could use a wrapper
-# for that. Or pam_env. Or whatever fits in the local setup. :)
-BASEDIR=${BASEDIR:-"$(readlink -f $(dirname "$0")/..)"}
-
-# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
-# by Joerg Jaspert and the Debian mirroradm group.
-# This is used to track which mirror is using which script version.
-VERSION="20170204"
-
-# Source our common functions
-. "${BASEDIR}/bin/common"
-
-########################################################################
-########################################################################
-## functions ##
-########################################################################
-########################################################################
-# We want to be able to get told what kind of sync we should do. This
-# might be anything, from the archive to sync, the stage to do, etc. A
-# list of currently understood and valid options is below. Multiple
-# options are seperated by space. All the words have to have the word
-# sync: in front or nothing will get used!
-#
-# Option Behaviour
-# stage1 Only do stage1 sync
-# stage2 Only do stage2 sync
-# all Do a complete sync
-# mhop Do a mhop sync, usually additionally to stage1
-# archive:foo Sync archive foo (if config for foo is available)
-# callback Call back when done (needs proper ssh setup for this to
-# work). It will always use the "command" callback:$HOSTNAME
-# where $HOSTNAME is the one defined below/in config and
-# will happen before slave mirrors are triggered.
-#
-# So to get us to sync all of the archive behind bpo and call back when
-# we are done, a trigger command of
-# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
-# trick.
-check_commandline() {
- while [[ $# -gt 0 ]]; do
- case "$1" in
- sync:stage1)
- SYNCSTAGE1="true"
- SYNCALL="false"
- ;;
- sync:stage2)
- SYNCSTAGE2="true"
- SYNCALL="false"
- ;;
- sync:callback)
- SYNCCALLBACK="true"
- ;;
- sync:archive:*)
- ARCHIVE=${1##sync:archive:}
- # We do not like / or . in the remotely supplied archive name.
- ARCHIVE=${ARCHIVE//\/}
- ARCHIVE=${ARCHIVE//.}
- ;;
- sync:all)
- SYNCALL="true"
- ;;
- sync:mhop)
- SYNCMHOP="true"
- ;;
- *)
- echo "Unknown option ${1} ignored"
- ;;
- esac
- shift # Check next set of parameters.
- done
-}
-
-# All the stuff we want to do when we exit, no matter where
-cleanup() {
- trap - ERR TERM HUP INT QUIT EXIT
- # all done. Mail the log, exit.
- log "Mirrorsync done";
-
- if [[ -n ${MAILTO} ]]; then
- # In case rsync had something on stderr
- if [[ -s ${LOGDIR}/rsync-${NAME}.error ]]; then
- mail -E -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date
+"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
- fi
- if [[ ${ERRORSONLY} = false ]]; then
- # And the normal log
- MAILFILES="${LOG}"
- if [[ ${FULLLOGS} = true ]]; then
- # Someone wants full logs including rsync
- MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
- fi
- cat ${MAILFILES} | mail -E -s "[${PROGRAM}@$(hostname -s)] archive
sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
- fi
- fi
-
- savelog "${LOGDIR}/rsync-${NAME}.log"
- savelog "${LOGDIR}/rsync-${NAME}.error"
- savelog "$LOG" > /dev/null
-
- rm -f "${LOCK}"
-}
-
-# Check rsyncs return value
-check_rsync() {
- ret=$1
- msg=$2
-
- # Lets get a statistical value
- SPEED="unknown"
- if [[ -f ${LOGDIR}/rsync-${NAME}.log ]]; then
- SPEED=$(
- SPEEDLINE=$(egrep '[0-9.,]+ bytes/sec'
"${LOGDIR}/rsync-${NAME}.log" | tail -n 1)
- set "nothing" ${SPEEDLINE}
- echo ${8:-""}
- )
- if [[ -n ${SPEED} ]]; then
- SPEED=${SPEED%%.*}
- SPEED=${SPEED//,}
- SPEED=$(( $SPEED / 1024 ))
- fi
- fi
- log "Latest recorded rsync transfer speed: ${SPEED} KB/s"
-
- # 24 - vanished source files. Ignored, that should be the target of
$UPDATEREQUIRED
- # and us re-running. If it's not, uplink is broken anyways.
- case "${ret}" in
- 0) return 0;;
- 24) return 0;;
- 23) return 2;;
- 30) return 2;;
- *)
- error "ERROR: ${msg}"
- return 1
- ;;
- esac
-}
-
-# Write a tracefile
-tracefile() {
- local TRACEFILE=${1:-"${TO}/${TRACE}"}
-
- case ${EXTENDEDTRACE} in
- none)
- log "No trace file wanted. Not creating one"
- ;;
- touch)
- log "Just touching the trace file"
- touch "${TRACEFILE}"
- ;;
- date|terse|full)
- log "Creating a ${EXTENDEDTRACE} trace file"
- tracedir=$(dirname "${TRACEFILE}")
- if [[ -d ${tracedir} ]]; then
- {
- LC_ALL=POSIX LANG=POSIX date -u
- if [[ date != ${EXTENDEDTRACE} ]]; then
- rfc822date=$(LC_ALL=POSIX LANG=POSIX date -u -R)
- echo "Date: ${rfc822date}"
- echo "Date-Started: ${DATE_STARTED}"
-
- FILESOA=$(awk -F": " ' $1=="Archive serial" {print
$2}' ${TO}/project/trace/master || echo unknown )
- echo "Archive serial: ${FILESOA}"
-
- echo "Used ftpsync version: ${VERSION}"
- echo "Running on host: ${TRACEHOST}"
- if [[ full = ${EXTENDEDTRACE} ]]; then
- ARCH=$((find ${TO}/dists -name Release -exec sed
-ne 's/^Architecture: //p' '{}' + 2>/dev/null || echo "unknown") | sort -u | tr
'\n' ' ')
- echo "Architectures: ${ARCH}"
- if [[ ${ARCH_INCLUDE} ]]; then
- echo "Architectures-Configuration: INCLUDE
$(tr ' ' '\n' <<< ${ARCH_INCLUDE} | sort -u | tr '\n' ' ')"
- elif [[ ${ARCH_EXCLUDE} ]]; then
- echo "Architectures-Configuration: EXCLUDE
$(tr ' ' '\n' <<< ${ARCH_EXCLUDE} | sort -u | tr '\n' ' ')"
- else
- echo "Architectures-Configuration: ALL"
- fi
- echo "Upstream-mirror: ${RSYNC_HOST}"
- echo "SSL: ${RSYNC_SSL}"
- total=0
- if [[ -e ${LOGDIR}/rsync-${NAME}.log ]]; then
- for bytes in $(awk -F': ' '$1 == "Total
bytes received" {print $2} ' "${LOGDIR}/rsync-${NAME}.log"); do
- bytes=${bytes//,}
- total=$(( total + bytes ))
- done
- echo "Total bytes received in rsync: ${total}"
- fi
- total_time=$(( STATS_TOTAL_RSYNC_TIME1 +
STATS_TOTAL_RSYNC_TIME2 ))
- echo "Total time spent in stage1 rsync:
${STATS_TOTAL_RSYNC_TIME1}"
- echo "Total time spent in stage2 rsync:
${STATS_TOTAL_RSYNC_TIME2}"
- echo "Total time spent in rsync: ${total_time}"
- if [[ 0 != ${total_time} ]]; then
- rate=$(( total / total_time ))
- echo "Average rate: ${rate} B/s"
- fi
- fi # full trace
- fi # other traces
- } > "${TRACEFILE}.new"
- mv "${TRACEFILE}.new" "${TRACEFILE}"
-
- if [[ full = ${EXTENDEDTRACE} ]] || [[ -e
${TO}/${TRACEHIERARCHY}.mirror ]]; then
- {
- if [[ -e ${TO}/${TRACEHIERARCHY}.mirror ]]; then
- cat ${TO}/${TRACEHIERARCHY}.mirror
- fi
- echo "$(basename "${TRACEFILE}") ${MIRRORNAME}
${TRACEHOST} ${RSYNC_HOST}"
- } > "${TO}/${TRACEHIERARCHY}".new
- mv "${TO}/${TRACEHIERARCHY}".new "${TO}/${TRACEHIERARCHY}"
- cp "${TO}/${TRACEHIERARCHY}"
"${TO}/${TRACEHIERARCHY}.mirror"
-
- (cd ${tracedir} && ls -1rt) | grep -E -v '^_|^master$' >
"${TO}/${TRACELIST}"
- else
- rm -f "${TO}/${TRACEHIERARCHY}" "${TO}/${TRACELIST}"
- fi
- fi
- ;;
- *)
- error "Unsupported EXTENDEDTRACE value configured in
${BASEDIR}/etc/${NAME}.conf, please fix"
- ;;
- esac
-
-}
-
-arch_imexclude() {
- local param="$1" arch="$2"
- EXCLUDE="${EXCLUDE} --${param}=/dists/**/binary-${arch}/
--${param}=/dists/**/installer-${arch}/
--${param}=/dists/**/Contents-${arch}.gz
--${param}=/dists/**/Contents-udeb-${arch}.gz
--${param}=/dists/**/Contents-${arch}.diff/
--${param}=/indices/**/arch-${arch}.files
--${param}=/indices/**/arch-${arch}.list.gz --${param}=/pool/**/*_${arch}.deb
--${param}=/pool/**/*_${arch}.udeb --${param}=/pool/**/*_${arch}.changes"
-}
-
-arch_exclude() {
- arch_imexclude exclude "$1"
-}
-
-arch_include() {
- arch_imexclude include "$1"
-}
-
-# Learn which archs to include/exclude based on ARCH_EXCLUDE and ARCH_INCLUDE
-# settings.
-#
-# Updates SOURCE_EXCLUDE (if not set already), and sets
-# EXCLUDE (which might also have --include statementss
-# followed by a --exclude *_*.<things>.
-set_exclude_include_archs() {
- local exclude_sources=false
-
- if [[ -n "${ARCH_EXCLUDE}" ]] && [[ -n "${ARCH_INCLUDE}" ]]; then
- echo >&2 "ARCH_EXCLUDE and ARCH_INCLUDE are mutually exclusive. Set only
one."
- exit 1
- fi
-
- if [[ -n "${ARCH_EXCLUDE}" ]]; then
- for ARCH in ${ARCH_EXCLUDE}; do
- arch_exclude ${ARCH}
- if [[ ${ARCH} = source ]]; then
- exclude_sources=true
- fi
- done
- elif [[ -n "${ARCH_INCLUDE}" ]]; then
- local include_arch_all=false
- exclude_sources=true
- for ARCH in ${ARCH_INCLUDE}; do
- arch_include ${ARCH}
- if [[ ${ARCH} = source ]]; then
- exclude_sources=false
- else
- include_arch_all=true
- fi
- done
- if [[ true = ${include_arch_all} ]]; then
- arch_include all
- fi
- arch_exclude '*'
- fi
-
- # we do not want sources, and SOURCE_EXCLUDE is not set in the config yet.
- if [[ true = ${exclude_sources} ]] && [[ -z ${SOURCE_EXCLUDE} ]]; then
- SOURCE_EXCLUDE=" --exclude=/dists/**/source/ --exclude=/pool/**/*.tar.*
--exclude=/pool/**/*.diff.* --exclude=/pool/**/*.dsc "
- fi
-}
-
-########################################################################
-########################################################################
-
-
-# As what are we called?
-NAME="$(basename $0)"
-# The original command line arguments need to be saved!
-if [[ $# -gt 0 ]]; then
- ORIGINAL_COMMAND=$*
-else
- ORIGINAL_COMMAND=""
-fi
-
-SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
-# Now, check if we got told about stuff via ssh
-if [[ -n ${SSH_ORIGINAL_COMMAND} ]]; then
- # We deliberately add "nothing" and ignore it right again, to avoid
- # people from outside putting some set options in the first place,
- # making us parse them...
- set "nothing" "${SSH_ORIGINAL_COMMAND}"
- shift
- # Yes, unqouted $* here. Or the function will only see it as one
- # parameter, which doesnt help the case in it.
- check_commandline $*
-fi
-
-# Now, we can locally override all the above variables by just putting
-# them into the .ssh/authorized_keys file forced command.
-if [[ -n ${ORIGINAL_COMMAND} ]]; then
- set ${ORIGINAL_COMMAND}
- check_commandline $*
-fi
-
-# If we have been told to do stuff for a different archive than default,
-# set the name accordingly.
-ARCHIVE=${ARCHIVE:-""}
-if [[ -n ${ARCHIVE} ]]; then
- NAME="${NAME}-${ARCHIVE}"
-fi
-
-# Now source the config for the archive we run on.
-# (Yes, people can also overwrite the options above in the config file
-# if they want to)
-if ! read_config "${NAME}.conf"; then
- echo "Nono, you can't tell us about random archives. Bad boy!"
- exit 1
-fi
-
-########################################################################
-# There should be nothing to edit here, use the config file #
-########################################################################
-MIRRORNAME=${MIRRORNAME:-$(hostname -f)}
-# Where to put logfiles in
-LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
-# Our own logfile
-LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
-
-# Where should we put all the mirrored files?
-TO=${TO:-"/srv/mirrors/debian/"}
-
-# used by log() and error()
-PROGRAM=${PROGRAM:-"${NAME}"}
-
-# Where to send mails about mirroring to?
-if [[ $(hostname -d) != debian.org ]]; then
- # We are not on a debian.org host
- MAILTO=${MAILTO:-"root"}
-else
- # Yay, on a .debian.org host
- MAILTO=${MAILTO:-"mirrorl...@debian.org"}
-fi
-# Want errors only or every log?
-ERRORSONLY=${ERRORSONLY:-"true"}
-# Want full logs, ie. including the rsync one?
-FULLLOGS=${FULLLOGS:-"false"}
-
-# How many logfiles to keep
-LOGROTATE=${LOGROTATE:-14}
-
-# Our lockfile
-LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
-# timeout for the lockfile, in case we have bash older than v4 (and no /proc)
-LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
-# sleeping time when an AUIP file is found but is not ours
-UIPSLEEP=${UIPSLEEP:-1200}
-# retries whenever an upstream (or possibly stale) AUIP file is found
-UIPRETRIES=${UIPRETRIES:-3}
-# Do we need another rsync run?
-UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
-# Trace file for mirror stats and checks (make sure we get full hostname)
-TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}
-TRACEHIERARCHY=${TRACEHIERARCHY:-"project/trace/_hierarchy"}
-TRACELIST=${TRACELIST:-"project/trace/_traces"}
-# The trace file can have different format/contents. Here you can select
-# what it will be.
-# Possible values are
-# "full" - all information
-# "terse" - partial, ftpsync version and local hostname
-# "date" - basic, timestamp only (date -u)
-# "touch" - just touch the file in existance
-# "none" - no tracefile at all
-#
-# Default and required value for Debian mirrors is full.
-EXTENDEDTRACE=${EXTENDEDTRACE:-"full"}
-
-# The local hostname to be written to the trace file.
-TRACEHOST=${TRACEHOST:-$(hostname -f)}
-
-# rsync program
-RSYNC=${RSYNC:-rsync}
-# Rsync filter rules. Used to protect various files we always want to keep,
even if we otherwise delete
-# excluded files
-RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME}
--filter=protect_${TRACE} --filter=protect_${TRACE}-stage1
--exclude=${TRACEHIERARCHY} --filter=protect_${TRACEHIERARCHY}
--exclude=${TRACELIST} --filter=protect_${TRACELIST}
--filter=protect_Archive-Update-Required-${MIRRORNAME}"}
-# limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited
-RSYNC_BW=${RSYNC_BW:-0}
-RSYNC_PROTOCOL=$(rsync_protocol)
-
-# Set the delete method to --delete-delay if protocol version is 30 or
-# greater (meaning rsync 3.0.0 or greater is used). Use --delete-after
-# otherwise.
-if [[ 30 -le $RSYNC_PROTOCOL ]]; then
- RSYNC_DELETE_METHOD=delay
-else
- RSYNC_DELETE_METHOD=after
-fi
-
-# Default rsync options for *every* rsync call
-RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats
${RSYNC_FILTER}"}
-# Options we only use in the first pass, where we do not want packages/sources
to fly in yet and don't want to delete files
-RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude=Packages* --exclude=Sources*
--exclude=Release* --exclude=InRelease --include=i18n/by-hash/**
--exclude=i18n/* --exclude=ls-lR*"}
-# Options for the second pass, where we do want everything, including deletion
of old and now unused files
-RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete
--delete-excluded"}
-# Which rsync share to use on our upstream mirror?
-RSYNC_PATH=${RSYNC_PATH:-"debian"}
-
-# Extra rsync options as defined by the admin locally. Won't be set
-# to any default by ftpsync. Those will be added to EACH AND EVERY rsync call.
-RSYNC_EXTRA=${RSYNC_EXTRA:-""}
-
-# Now add the bwlimit option. As default is 0 we always add it, rsync
interprets
-# 0 as unlimited, so this is safe.
-RSYNC_OPTIONS="${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
-
-# Finally, make sure RSYNC_OPTIONS2 has either --delete-after or --delete-delay
-RSYNC_OPTION_REGEX="--delete-(after|delay)"
-if ! [[ ${RSYNC_OPTIONS2} =~ ${RSYNC_OPTION_REGEX} ]]; then
- RSYNC_OPTIONS2+=" --delete-${RSYNC_DELETE_METHOD}"
-fi
-unset RSYNC_OPTION_REGEX
-
-# We have no default host to sync from, but will error out if its unset
-RSYNC_HOST=${RSYNC_HOST:-""}
-# Error out if we have no host to sync from
-if [[ -z ${RSYNC_HOST} ]]; then
- error "Missing a host to mirror from, please set RSYNC_HOST variable in
${BASEDIR}/etc/${NAME}.conf"
-fi
-
-# our username for the rsync share
-RSYNC_USER=${RSYNC_USER:-""}
-# the password
-RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
-
-# a possible proxy
-RSYNC_PROXY=${RSYNC_PROXY:-""}
-
-# Do we sync stage1?
-SYNCSTAGE1=${SYNCSTAGE1:-"false"}
-# Do we sync stage2?
-SYNCSTAGE2=${SYNCSTAGE2:-"false"}
-# Do we sync all?
-SYNCALL=${SYNCALL:-"true"}
-# Do we have a mhop sync?
-SYNCMHOP=${SYNCMHOP:-"false"}
-# Do we callback? (May get changed later)
-SYNCCALLBACK=${SYNCCALLBACK:-"false"}
-# If we call back we need some more options defined in the config file.
-CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
-CALLBACKHOST=${CALLBACKHOST:-"none"}
-CALLBACKKEY=${CALLBACKKEY:-"none"}
-
-# General excludes. Don't list architecture specific stuff here, use
ARCH_EXCLUDE for that!
-EXCLUDE=${EXCLUDE:-""}
-
-# collect some stats
-STATS_TOTAL_RSYNC_TIME1=0
-STATS_TOTAL_RSYNC_TIME2=0
-
-# The temp directory used by rsync --delay-updates is not
-# world-readable remotely. Always exclude it to avoid errors.
-EXCLUDE="${EXCLUDE} --exclude=.~tmp~/"
-
-SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""}
-ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
-ARCH_INCLUDE=${ARCH_INCLUDE:-""}
-
-RSYNC_SSL=${RSYNC_SSL:-"false"}
-RSYNC_SSL_PORT=${RSYNC_SSL_PORT:-"1873"}
-RSYNC_SSL_CAPATH=${RSYNC_SSL_CAPATH:-"/etc/ssl/certs"}
-RSYNC_SSL_METHOD=${RSYNC_SSL_METHOD:-"stunnel4"}
-
-if [[ true != ${RSYNC_SSL} ]]; then
- RSYNC_SSL_OPTIONS=""
-else
- export RSYNC_SSL_PORT
- export RSYNC_SSL_CAPATH
- export RSYNC_SSL_METHOD
- RSYNC_SSL_OPTIONS="-e ${BASEDIR}/bin/rsync-ssl-tunnel"
-fi
-
-# Hooks
-HOOK1=${HOOK1:-""}
-HOOK2=${HOOK2:-""}
-HOOK3=${HOOK3:-""}
-HOOK4=${HOOK4:-""}
-HOOK5=${HOOK5:-""}
-
-# Are we a hub?
-HUB=${HUB:-"false"}
-
-########################################################################
-# Really nothing to see below here. Only code follows. #
-########################################################################
-########################################################################
-DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R)
-
-# Exclude architectures defined in $ARCH_EXCLUDE
-set_exclude_include_archs
-
-# Some sane defaults
-cd "${BASEDIR}"
-umask 022
-
-# If we are here for the first time, create the
-# destination and the trace directory
-mkdir -p "${TO}/project/trace"
-
-# Used to make sure we will have the archive fully and completly synced before
-# we stop, even if we get multiple pushes while this script is running.
-# Otherwise we can end up with a half-synced archive:
-# - get a push
-# - sync, while locked
-# - get another push. Of course no extra sync run then happens, we are locked.
-# - done. Archive not correctly synced, we don't have all the changes from the
second push.
-touch "${UPDATEREQUIRED}"
-
-# Check to see if another sync is in progress
-if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
- if [[ ${BASH_VERSINFO[0]} -gt 3 ]] || [[ -L /proc/self ]]; then
- # We have a recent enough bash version, lets do it the easy way,
- # the lock will contain the right pid, thanks to $BASHPID
- if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then
- # Process does either not exist or is not owned by us.
- echo "$$" > "${LOCK}"
- else
- echo "Unable to start rsync, lock file still exists, PID $(<
${LOCK})"
- exit 1
- fi
- else
- # Old bash, means we dont have the right pid in our lockfile
- # So take a different way - guess if it is still there by comparing
its age.
- # Not optimal, but hey.
- stamptime=$(date --reference="${LOCK}" +%s)
- unixtime=$(date +%s)
- difference=$(( $unixtime - $stamptime ))
- if [[ ${difference} -ge ${LOCKTIMEOUT} ]]; then
- # Took longer than LOCKTIMEOUT minutes? Assume it broke and take
the lock
- echo "$$" > "${LOCK}"
- else
- echo "Unable to start rsync, lock file younger than one hour"
- exit 1
- fi
- fi
-fi
-
-# When we exit normally we call cleanup on our own. Otherwise we want it
called by
-# this trap. (We can not trap on EXIT, because that is called when the main
script
-# exits. Which also happens when we background the mainroutine, ie. while we
still
-# run!)
-trap cleanup ERR TERM HUP INT QUIT
-
-# Start log by redirecting stdout and stderr there and closing stdin
-mkdir -p "$LOGDIR"
-exec >"$LOG" 2>&1 <&-
-log "Mirrorsync start"
-
-# Look who pushed us and note that in the log.
-SSH_CONNECTION=${SSH_CONNECTION:-""}
-PUSHFROM="${SSH_CONNECTION%%\ *}"
-if [[ -n ${PUSHFROM} ]]; then
- log "We got pushed from ${PUSHFROM}"
-fi
-
-if [[ true = ${SYNCCALLBACK} ]]; then
- if [[ none = ${CALLBACKHOST} ]] || [[ none = ${CALLBACKKEY} ]]; then
- SYNCCALLBACK="false"
- error "We are asked to call back, but we do not know where to and do
not have a key, ignoring callback"
- fi
-fi
-
-HOOK=(
- HOOKNR=1
- HOOKSCR=${HOOK1}
-)
-hook $HOOK
-
-# Now, we might want to sync from anonymous too.
-# This is that deep in this script so hook1 could, if wanted, change things!
-if [[ -z ${RSYNC_USER} ]]; then
- RSYNCPTH="${RSYNC_HOST}"
-else
- RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
-fi
-
-# Now do the actual mirroring, and run as long as we have an updaterequired
file.
-export RSYNC_PASSWORD
-export RSYNC_PROXY
-
-UPDATE_RETRIES=0
-
-while [[ -e ${UPDATEREQUIRED} ]]; do
- log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
-
- # if we want stage1 *or* all
- if [[ true = ${SYNCSTAGE1} ]] || [[ true = ${SYNCALL} ]]; then
- while [[ -e ${UPDATEREQUIRED} ]]; do
- rm -f "${UPDATEREQUIRED}"
- log "Running stage1: ${RSYNC} ${RSYNC_SSL_OPTIONS}
${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE}
${RSYNCPTH}::${RSYNC_PATH} ${TO}"
-
- set +e
- # Step one, sync everything except Packages/Releases
- rsync_started=$(date +%s)
- ${RSYNC} ${RSYNC_SSL_OPTIONS} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1}
${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}"
>>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
- result=$?
- rsync_ended=$(date +%s)
- STATS_TOTAL_RSYNC_TIME1=$(( STATS_TOTAL_RSYNC_TIME1 + rsync_ended
- rsync_started ))
- set -e
-
- log "Back from rsync with returncode ${result}"
- done
- else
- # Fake a good resultcode
- result=0
- fi # Sync stage 1?
- rm -f "${UPDATEREQUIRED}"
-
- set +e
- check_rsync $result "Sync step 1 went wrong, got errorcode ${result}.
Logfile: ${LOG}"
- GO=$?
- set -e
- if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then
- log "We got error ${result} from rsync, but a second push went in
hence ignoring this error for now"
- elif [[ ${GO} -ne 0 ]]; then
- exit 3
- fi
-
- HOOK=(
- HOOKNR=2
- HOOKSCR=${HOOK2}
- )
- hook $HOOK
-
- # if we want stage2 *or* all
- if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then
- upstream_uip=false
- for aupfile in "${TO}/Archive-Update-in-Progress-"*; do
- case "$aupfile" in
- "${TO}/Archive-Update-in-Progress-*")
- error "Lock file is missing, this should not happen"
- ;;
- "${LOCK}")
- :
- ;;
- *)
- if [[ -f $aupfile ]]; then
- # Remove the file, it will be synced again if
- # upstream is still not done
- rm -f "$aupfile"
- else
- log "AUIP file '$aupfile' is not really a file, weird"
- fi
- upstream_uip=true
- ;;
- esac
- done
-
- if [[ true = ${upstream_uip} ]]; then
- log "Upstream archive update in progress, skipping stage2"
- if [[ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]]; then
- log "Retrying update in ${UIPSLEEP}"
- touch "${UPDATEREQUIRED}"
- UPDATE_RETRIES=$(($UPDATE_RETRIES+1))
- sleep "${UIPSLEEP}"
- result=0
- else
- error "Update has been retried ${UPDATE_RETRIES} times,
aborting"
- log "Perhaps upstream is still updating or there's a stale
AUIP file"
- result=1
- fi
- else
- log "Running stage2: ${RSYNC} ${RSYNC_SSL_OPTIONS}
${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE}
${RSYNCPTH}::${RSYNC_PATH} ${TO}"
-
- set +e
- # We are lucky, it worked. Now do step 2 and sync again, this time
including
- # the packages/releases files
- rsync_started=$(date +%s)
- ${RSYNC} ${RSYNC_SSL_OPTIONS} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2}
${EXCLUDE} ${SOURCE_EXCLUDE} \
- ${RSYNCPTH}::${RSYNC_PATH} "${TO}"
>>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
- result=$?
- rsync_ended=$(date +%s)
- STATS_TOTAL_RSYNC_TIME2=$(( STATS_TOTAL_RSYNC_TIME2 + rsync_ended
- rsync_started ))
- set -e
-
- log "Back from rsync with returncode ${result}"
- fi
- else
- # Fake a good resultcode
- result=0
- fi # Sync stage 2?
-
- set +e
- check_rsync $result "Sync step 2 went wrong, got errorcode ${result}.
Logfile: ${LOG}"
- GO=$?
- set -e
- if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then
- log "We got error ${result} from rsync, but a second push went in
hence ignoring this error for now"
- elif [[ ${GO} -ne 0 ]]; then
- exit 4
- fi
-
- HOOK=(
- HOOKNR=3
- HOOKSCR=${HOOK3}
- )
- hook $HOOK
-done
-
-# We only update our tracefile when we had a stage2 or an all sync.
-# Otherwise we would update it after stage1 already, which is wrong.
-if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then
- tracefile
- if [[ true = ${SYNCALL} ]]; then
- rm -f "${TO}/${TRACE}-stage1"
- fi
-elif [[ true = ${SYNCSTAGE1} ]]; then
- tracefile "${TO}/${TRACE}-stage1"
-fi
-
-
-HOOK=(
- HOOKNR=4
- HOOKSCR=${HOOK4}
-)
-hook $HOOK
-
-if [[ true = ${SYNCCALLBACK} ]]; then
- set +e
- callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
- set -e
-fi
-
-# Remove the Archive-Update-in-Progress file before we push our downstreams.
-rm -f "${LOCK}"
-
-# Check if there is a newer version of ftpsync. If so inform the admin, but not
-# more than once every third day.
-if [[ -r ${TO}/project/ftpsync/LATEST.VERSION ]]; then
- LATEST=$(< "${TO}/project/ftpsync/LATEST.VERSION")
- if ! [[ ${LATEST} =~ [0-9]+ ]]; then
- LATEST=0
- fi
- if [[ ${LATEST} -gt ${VERSION} ]]; then
- if [[ -n ${MAILTO} ]]; then
- difference=0
- if [[ -f ${LOGDIR}/ftpsync.newversion ]]; then
- stamptime=$(< "${LOGDIR}/ftpsync.newversion")
- unixtime=$(date +%s)
- difference=$(( $unixtime - $stamptime ))
- fi
- if [[ ${difference} -ge 259200 ]]; then
- # Only warn every third day
- mail -s "[$(hostname -s)] Update for ftpsync available"
${MAILTO} <<EOF
-Hello admin,
-
-i found that there is a new version of me available.
-Me lonely ftpsync is currently version: ${VERSION}
-New release of myself is available as: ${LATEST}
-
-Me, myself and I - and the Debian mirroradmins - would be very grateful
-if you could update me. You can find the latest version on your mirror,
-check $(hostname -s):${TO}/project/ftpsync/ftpsync-${LATEST}.tar.gz
-
-You can ensure the validity of that file by using sha512sum or md5sum
-against the available checksum files secured with a signature from the
-Debian FTPMaster signing key.
-
-EOF
-
- date +%s > "${LOGDIR}/ftpsync.newversion"
- fi
- fi
- else
- # Remove a possible stampfile
- rm -f "${LOGDIR}/ftpsync.newversion"
- fi
-fi
-
-if [[ ${HUB} = true ]]; then
- # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
- if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]] || [[ true =
${SYNCMHOP} ]]; then
- RUNMIRRORARGS=""
- if [[ -n ${ARCHIVE} ]]; then
- # We tell runmirrors about the archive we are running on.
- RUNMIRRORARGS="-a ${ARCHIVE}"
- fi
- # We also tell runmirrors that we are running it from within ftpsync,
so it can change
- # the way it works with mhop based on that.
- RUNMIRRORARGS="${RUNMIRRORARGS} -f"
-
- if [[ true = ${SYNCSTAGE1} ]]; then
- # This is true when we have a mhop sync. A normal multi-stage push
sending stage1 will
- # not get to this point.
- # So if that happens, tell runmirrors we are doing mhop
- RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
- elif [[ true = ${SYNCSTAGE2} ]]; then
- RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
- elif [[ true = ${SYNCALL} ]]; then
- RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
- fi
- log "Trigger slave mirrors using ${RUNMIRRORARGS}"
- ${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
- log "Trigger slave done"
-
- HOOK=(
- HOOKNR=5
- HOOKSCR=${HOOK5}
- )
- hook $HOOK
- fi
-fi
-
-# All done, lets call cleanup
-cleanup
diff --git a/modules/mirrors/files/ssh-debian-archvsync.pub
b/modules/mirrors/files/ssh-debian-archvsync.pub
index 19b2a70..ae92c21 100644
--- a/modules/mirrors/files/ssh-debian-archvsync.pub
+++ b/modules/mirrors/files/ssh-debian-archvsync.pub
@@ -1 +1 @@
-no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,no-user-rc,command="/var/lib/mirror/archvsync/bin/ftpsync"
ssh-rsa
AAAAB3NzaC1yc2EAAAADAQABAAABAQDNZoV9j9KFMFssqLLbKF0b35npFAE3USHgtyN3pUsFm8Wsh1i4ZVly+8Nks7LVJrAI8uCrfvZKyQrqdcjAyy7UAGXfz5YrMxj0PZbvz9rHWa7Hxl+wO5mYwtklN8ek7uFgqDCmMOkE+4uIxOlPzDJ95BUBx5SVm8jAaMmaBoX20rxO5BGvguVU4RJ1V9EZKB4B9pGbdScRJOWAGUrg8HSKJGtlNQTF0xIU6P6quF0O+qpjF9CiB2GVzRJZXNpAhlekpVx5/Xg6N+sIN7QxxHViO7xwTQgteUehbrpToNc1N4jf4kciHmP/jdgEus/W61U410QfXVUYKOTDuGzAotl/
archvs...@syncproxy.cna.debian.org (aka mirror-umn.debian.org; 2015-12-09)
+no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,no-user-rc,command="/usr/bin/ftpsync"
ssh-rsa
AAAAB3NzaC1yc2EAAAADAQABAAABAQDNZoV9j9KFMFssqLLbKF0b35npFAE3USHgtyN3pUsFm8Wsh1i4ZVly+8Nks7LVJrAI8uCrfvZKyQrqdcjAyy7UAGXfz5YrMxj0PZbvz9rHWa7Hxl+wO5mYwtklN8ek7uFgqDCmMOkE+4uIxOlPzDJ95BUBx5SVm8jAaMmaBoX20rxO5BGvguVU4RJ1V9EZKB4B9pGbdScRJOWAGUrg8HSKJGtlNQTF0xIU6P6quF0O+qpjF9CiB2GVzRJZXNpAhlekpVx5/Xg6N+sIN7QxxHViO7xwTQgteUehbrpToNc1N4jf4kciHmP/jdgEus/W61U410QfXVUYKOTDuGzAotl/
archvs...@syncproxy.cna.debian.org (aka mirror-umn.debian.org; 2015-12-09)
diff --git a/modules/mirrors/manifests/debian.pp
b/modules/mirrors/manifests/debian.pp
index 3b72275..ff5ce64 100644
--- a/modules/mirrors/manifests/debian.pp
+++ b/modules/mirrors/manifests/debian.pp
@@ -23,38 +23,12 @@
mode => '0755',
}
- # this is <https://ftp-master.debian.org/git/archvsync.git>
- # right now we just ship bin/ftpsync & bin/common
- # there is soon going to be a Debian package, use that then instead
- file { "${mirrors::homedir}/archvsync":
- ensure => directory,
- recurse => true,
- purge => true,
- owner => 'mirror',
- group => 'mirror',
- mode => '0755',
- source => 'puppet:///modules/mirrors/archvsync',
- }
-
- # don't purge logs (you'd expect more from people that love the FHS...)
- file { "${mirrors::homedir}/archvsync/log":
- ensure => directory,
- purge => false,
- owner => 'mirror',
- group => 'mirror',
- mode => '0755',
- }
-
- # move to .config/archvsync or /etc/archvsync?
- file { "${mirrors::homedir}/archvsync/etc":
- ensure => directory,
- owner => 'mirror',
- group => 'mirror',
- mode => '0755',
+ package { 'ftpsync':
+ ensure => present,
}
# this is our configuration for archvsync
- file { "${mirrors::homedir}/archvsync/etc/ftpsync.conf":
+ file { '/etc/ftpsync':
ensure => present,
owner => 'mirror',
group => 'mirror',
@@ -62,6 +36,13 @@
content => template('mirrors/ftpsync.conf.erb'),
}
+ file { '/var/log/ftpsync':
+ ensure => directory,
+ owner => 'mirror',
+ group => 'mirror',
+ mode => '0755',
+ }
+
# allow the Debian syncproxy to trigger ftpsync runs over ssh
ssh::userkey { 'mirror':
source => 'puppet:///modules/mirrors/ssh-debian-archvsync.pub',
diff --git a/modules/mirrors/spec/classes/mirrors_debian_spec.rb
b/modules/mirrors/spec/classes/mirrors_debian_spec.rb
index 88d7159..ed01c24 100644
--- a/modules/mirrors/spec/classes/mirrors_debian_spec.rb
+++ b/modules/mirrors/spec/classes/mirrors_debian_spec.rb
@@ -10,16 +10,7 @@
})
end
it do
- should contain_file('/var/lib/mirror/archvsync/').with({
- 'ensure' => 'directory',
- 'owner' => 'mirror',
- 'group' => 'mirror',
- 'mode' => '0755',
- 'source' => 'puppet:///modules/mirrors/archvsync',
- })
- end
- it do
- should
contain_file('/var/lib/mirror/archvsync/etc/ftpsync.conf').with({
+ should contain_file('/etc/ftpsync').with({
'ensure' => 'present',
'owner' => 'mirror',
'group' => 'mirror',
@@ -27,7 +18,11 @@
})
end
it do
- # 18eaf6c70 changed to push mirroring
- should_not contain_cron('update-debian-mirror')
+ should contain_file('/var/log/ftpsync/').with({
+ 'ensure' => 'directory',
+ 'owner' => 'mirror',
+ 'group' => 'mirror',
+ 'mode' => '0755',
+ })
end
end
diff --git a/modules/mirrors/templates/ftpsync.conf.erb
b/modules/mirrors/templates/ftpsync.conf.erb
index 322c8c2..844fe2a 100644
--- a/modules/mirrors/templates/ftpsync.conf.erb
+++ b/modules/mirrors/templates/ftpsync.conf.erb
@@ -7,5 +7,6 @@
MAILTO="root"
ERRORSONLY="true"
+LOGDIR="/var/log/ftpsync"
FULLLOGS="false"
LOGROTATE=14
--
To view, visit https://gerrit.wikimedia.org/r/384791
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I1cca0f806a014b5c05f7296b78eda7c2004c79a9
Gerrit-PatchSet: 2
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Faidon Liambotis <fai...@wikimedia.org>
Gerrit-Reviewer: Faidon Liambotis <fai...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
