Le vendredi 20 avril à 00h 16mn 13s (+0200), Raphael Hertzog a écrit : > > IMO the special casing for ports.debian.org architectures should be > handled in a dedicated wrapper. And maybe debootstrap needs new features > to make this wrapper possible. >
Hi, It would be beneficial to add a new option to debootstrap to postprocess downloaded package indices with an external helper program. This would allow to enforce the policy : 'debootstrap should install the newest version of each package' Please find below a first try to implement it. I would be thankful for comments, advice or suggestions of improvement. Regards, JH Chatenet diff -Naur a/debootstrap/debootstrap b/debootstrap/debootstrap --- a/debootstrap/debootstrap 2020-03-13 03:03:08.000000000 +0100 +++ b/debootstrap/debootstrap 2020-07-20 11:08:15.531224245 +0200 @@ -46,6 +46,7 @@ CHECKCERTIF="" PRIVATEKEY="" CACHE_DIR="" +PACKAGES_FILES_PROCESSOR="" DEF_MIRROR="http://deb.debian.org/debian"; DEF_HTTPS_MIRROR="https://deb.debian.org/debian"; @@ -125,6 +126,10 @@ (requires --second-stage) --extractor=TYPE override automatic .deb extractor selection (supported: $EXTRACTORS_SUPPORTED) + --packages-files-processor=HELPER + specifies the path to an external helper program + to postprocess downloaded Packages files + (this may be beneficial in conjunction with --extra-suites) --debian-installer used for internal purposes by debian-installer --private-key=file read the private key from file --certificate=file use the client certificate stored in file (PEM) @@ -323,6 +328,17 @@ fi EXTRA_SUITES="$(echo "$EXTRA_SUITES" | tr , " ")" ;; + --packages-files-processor|--packages-files-processor=?*) + if [ "$1" = "--packages-files-processor" ] && [ -n "$2" ]; then + PACKAGES_FILES_PROCESSOR="$2" + shift 2 + elif [ "$1" != "${1#--packages-files-processor=}" ]; then + PACKAGES_FILES_PROCESSOR="${1#--packages-files-processor=}" + shift 1 + else + error 1 NEEDARG "option requires an argument %s" "$1" + fi + ;; --components|--components=?*) if [ "$1" = "--components" ] && [ -n "$2" ]; then USE_COMPONENTS="$2" @@ -459,6 +475,9 @@ if [ -e "$DEBOOTSTRAP_DIR/extra-suites" ]; then EXTRA_SUITES=$(cat "$DEBOOTSTRAP_DIR/extra-suites") fi + if [ -e "$DEBOOTSTRAP_DIR/extra-suites-unprocessed" ]; then + extra_suites_unprocessed=$(cat "$DEBOOTSTRAP_DIR/extra-suites-unprocessed") + fi ARCH=$(cat "$DEBOOTSTRAP_DIR/arch") USER_MIRROR=$(cat "$DEBOOTSTRAP_DIR/mirror") if [ -e "$DEBOOTSTRAP_DIR/variant" ]; then @@ -686,6 +705,8 @@ ########################################################################### +packages_files_postprocess_initialize + if am_doing_phase finddebs; then if [ "$FINDDEBS_NEEDS_INDICES" = "true" ] || \ [ "$RESOLVE_DEPS" = "true" ]; then @@ -751,6 +772,8 @@ echo "$SUITE" >"$TARGET/debootstrap/suite" [ "" = "$EXTRA_SUITES" ] || echo "$EXTRA_SUITES" >"$TARGET/debootstrap/extra-suites" + [ "$EXTRA_SUITES" = "$extra_suites_unprocessed" ] || + echo "$extra_suites_unprocessed" >"$TARGET/debootstrap/extra-suites-unprocessed" [ "" = "$VARIANT" ] || echo "$VARIANT" >"$TARGET/debootstrap/variant" echo "$required" >"$TARGET/debootstrap/required" @@ -786,8 +809,10 @@ cp "$DEBOOTSTRAP_DIR"/scripts/*-common "$TARGET/debootstrap/" echo "$ARCH" >"$TARGET/debootstrap/arch" echo "$SUITE" >"$TARGET/debootstrap/suite" - [ "" = "$EXTRA_SUITES" ] || - echo "$EXTRA_SUITES" >"$TARGET/debootstrap/extra-suites" + if [ -n "$EXTRA_SUITES" ]; then + echo "$EXTRA_SUITES" >"$TARGET/debootstrap/extra-suites" + echo "$extra_suites_unprocessed" >"$TARGET/debootstrap/extra-suites-unprocessed" + fi echo "$USER_MIRROR" >"$TARGET/debootstrap/mirror" [ "" = "$VARIANT" ] || echo "$VARIANT" >"$TARGET/debootstrap/variant" diff -Naur a/debootstrap/debootstrap.8 b/debootstrap/debootstrap.8 --- a/debootstrap/debootstrap.8 2020-03-09 06:21:13.000000000 +0100 +++ b/debootstrap/debootstrap.8 2020-07-20 11:20:58.187006051 +0200 @@ -63,6 +63,10 @@ .IP "\fB\-\-extra-suites=alpha,beta\fP" Also use packages from the listed suites of the archive. .IP +.IP "\fB\-\-packages-files-processor=HELPER\fP" +Specifies the path to an external helper program to postprocess downloaded +Packages files. This may be beneficial in conjunction with --extra-suites. +.IP .IP "\fB\-\-components=alpha,beta\fP" Use packages from the listed components of the archive. .IP diff -Naur a/debootstrap/functions b/debootstrap/functions --- a/debootstrap/functions 2020-03-14 01:53:38.000000000 +0100 +++ b/debootstrap/functions 2020-07-20 11:16:16.485609169 +0200 @@ -728,10 +728,16 @@ if [ ! -f "$pkgdest" ]; then error 1 COULDNTDL "Couldn't download %s" "$m/$path" fi + + packages_files_postprocess_accumulate "$pkgdest" + donepkgs=$(($donepkgs + ${i#* })) progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files" done done + + packages_files_postprocess + } get_package_sizes () { @@ -807,7 +813,7 @@ leftoverdebs=$(printf "$leftoverdebs"|tr ' ' '\n'|sort -u|tr '\n' ' ') numdebs=$(printf "$leftoverdebs"|wc -w) - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in $COMPONENTS; do if [ "$countdebs" -ge "$numdebs" ]; then break; fi @@ -839,7 +845,7 @@ :>"$TARGET/debootstrap/debpaths" pkgs_to_get="$*" - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in $COMPONENTS; do path="dists/$s/$c/binary-$ARCH/Packages" for m in $MIRRORS; do @@ -880,9 +886,16 @@ elif get "$m/$path" "$pkgdest"; then true fi + + if [ -f "$pkgdest" ]; then + packages_files_postprocess_accumulate "$pkgdest" + fi done done done + + packages_files_postprocess + progress 100 100 DOWNMAINPKGS "Downloading Packages file" } @@ -892,7 +905,7 @@ :>"$TARGET/debootstrap/debpaths" for p in "$@"; do - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in $COMPONENTS; do local details="" for m in $MIRRORS; do @@ -934,7 +947,7 @@ field="$1" shift for m1 in $MIRRORS; do - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in $COMPONENTS; do path="dists/$s/$c/binary-$ARCH/Packages" pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" @@ -1532,7 +1545,7 @@ local ALLPKGS2=""; while [ "$PKGS" != "" ]; do local NEWPKGS="" - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do local path="dists/$s/$c/binary-$ARCH/Packages" local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" @@ -1541,7 +1554,7 @@ done PKGS=$(echo "$PKGS $NEWPKGS" | tr ' ' '\n' | sort | uniq) local REALPKGS="" - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do local path="dists/$s/$c/binary-$ARCH/Packages" local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" @@ -1560,7 +1573,7 @@ local m1 c path pkgdest pkg m1="${MIRRORS%% *}" - for s in $SUITE $EXTRA_SUITES; do + for s in $SUITE $extra_suites_unprocessed; do for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do path="dists/$s/$c/binary-$ARCH/Packages" pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" @@ -1881,3 +1894,76 @@ fakechroot" >> "$TARGET/var/lib/dpkg/diversions" } + +############################################################### 'Packages' files postprocessing + +packages_files_postprocess_initialize () { + + extra_suites_unprocessed="$EXTRA_SUITES" + postprocess_packages_files= + + if [ -n "$PACKAGES_FILES_PROCESSOR" ]; then + + if [ -n "$UNPACK_TARBALL" ]; then + warning TARBALL_PROC "option --packages-files-processor is ignored when unpacking a tarball" + elif [ -x "$PACKAGES_FILES_PROCESSOR" ]; then + postprocess_packages_files=true + else + error 1 NO_PKGPROC "Packages files postprocessor %s not found or not executable" "$PACKAGES_FILES_PROCESSOR" + fi + fi + + if [ -n "$UNPACK_TARBALL" ] && [ -e "$TARGET/debootstrap/extra-suites-unprocessed" ]; then + extra_suites_unprocessed=$(cat "$TARGET/debootstrap/extra-suites-unprocessed") + info UNPROCESSED_SET "using extra_suites_unprocessed=%s" "$extra_suites_unprocessed" + fi + +} + +packages_files_postprocess_accumulate () { + + if [ -n "$postprocess_packages_files" ]; then + local src dst path + src="$1" + path="dists/$SUITE/$c/binary-$ARCH/Packages" + dst="$TARGET/$($DLDEST pkg "$SUITE" "$c" "$ARCH" "$m" "$path").to_process" + + cat "$src" >> "$dst" + fi +} + + +packages_files_postprocess () { + + if [ -n "$postprocess_packages_files" ]; then + local m c src dst path + + for m in $MIRRORS; do + for c in $COMPONENTS; do + path="dists/$SUITE/$c/binary-$ARCH/Packages" + dst="$TARGET/$($DLDEST pkg "$SUITE" "$c" "$ARCH" "$m" "$path")" + src="${dst}.to_process" + + # Do nothing on empty Packages files + if [ -s "$src" ]; then + "$PACKAGES_FILES_PROCESSOR" "$src" > "${dst}.processed" + ret=$? + if [ $ret -ne 0 ] || [ ! -s "${dst}.processed" ]; then + error 1 COULDNTPROC "Couldn't postprocess %s: ret code=%s" "$dst" "$ret" + fi + + mv "$dst" "${dst}.unprocessed" + mv "${dst}.processed" "$dst" + fi + + if [ -f "$src" ]; then + # Cleaning up (required after --download-only) + rm -f "$src" + fi + done + done + # all packages appear in the Packages file of $SUITE after postprocessing + # there is no need to consider other Packages files + extra_suites_unprocessed= + fi +}
README.debian-ports.gz
Description: application/gzip
