Hello, This set of patches has been updated to apply to debmirror 1:2.4.4.
(Still haven't reworked the core batching changes, but these are all useful on their own...) Thanks! -Kees -- Kees Cook @debian.org
Description: increases rsync batching size, leaves off troublesome -I rsync flag, auto-flushes, and disables motd display during rsync. Author: Kees Cook <[email protected]> Bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=455082 Index: debmirror-2.4.4/debmirror =================================================================== --- debmirror-2.4.4.orig/debmirror 2010-03-25 11:13:29.000000000 -0700 +++ debmirror-2.4.4/debmirror 2010-04-05 11:36:47.764802267 -0700 @@ -311,7 +311,7 @@ =item --rsync-options=options Specify alternative rsync options to be used. Default options are -"-aIL --partial". Care must be taken when specifying alternative +"-aL --partial". Care must be taken when specifying alternative options not to disrupt operations, it's best to only add to those options. @@ -542,7 +542,7 @@ our $download_method="ftp"; our $timeout=300; our $max_batch=0; -our $rsync_batch=200; +our $rsync_batch=300; our $num_errors=0; our $bytes_to_get=0; our $bytes_gotten=0; @@ -553,7 +553,7 @@ our $start_time = time; our $dry_run=0; our $do_dry_run=0; -our $rsync_options="-aIL --partial"; +our $rsync_options="-aL --partial"; our $ignore_small_errors=0; our $diff_mode="use"; our $gzip_options="-9 -n --rsyncable"; @@ -563,6 +563,9 @@ my $HOME; ($HOME = $ENV{'HOME'}) or die "HOME not defined in environment!\n"; +# Switch to auto-flushing mode for stdout. +select STDOUT; $|=1; + # Load in config files require "/etc/debmirror.conf" if -r "/etc/debmirror.conf"; require "$HOME/.debmirror.conf" if -r "$HOME/.debmirror.conf"; @@ -1267,7 +1270,7 @@ print $fh "$_\n"; } } - system ("rsync --timeout=$timeout $opt $rsyncremote --include-from=$rsynctempfile --exclude='*' $mirrordir"); + system ("rsync --no-motd --timeout=$timeout $opt $rsyncremote --include-from=$rsynctempfile --exclude='*' $mirrordir"); close $fh; unlink $rsynctempfile; foreach my $dest (@result) { @@ -1691,7 +1694,7 @@ } $opt = "$opt --progress" if $progress; $opt = "$opt -v" if $debug; - system ("rsync --timeout=$timeout $opt $rsyncremote --include=$file --exclude='*' ."); + system ("rsync --no-motd --timeout=$timeout $opt $rsyncremote --include=$file --exclude='*' ."); if ($? == 0 && -f $file) { return 1; } else { @@ -1769,7 +1772,7 @@ for (@result) { print $fh "$_\n"; } - system ("rsync --timeout=$timeout $opt $rsyncremote --delete --include-from=$rsynctempfile --exclude='*' $mirrordir"); + system ("rsync --no-motd --timeout=$timeout $opt $rsyncremote --delete --include-from=$rsynctempfile --exclude='*' $mirrordir"); close $fh; unlink $rsynctempfile; }
Description: silence any errors from find. Author: Kees Cook <[email protected]> Bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=455082 Index: debmirror-2.4.4/debmirror =================================================================== --- debmirror-2.4.4.orig/debmirror 2010-04-05 11:36:47.000000000 -0700 +++ debmirror-2.4.4/debmirror 2010-04-05 11:37:46.462649134 -0700 @@ -1327,7 +1327,7 @@ say("Everything OK. Moving meta files."); chdir($tempdir) or die "unable to chdir($tempdir): $!\n"; my $res=0; -foreach my $file (`find . -type f`) { +foreach my $file (`find . -type f 2>/dev/null`) { chomp $file; $file=~s:^\./::; # this skips diff files if unwanted @@ -1401,7 +1401,7 @@ # makes directories.. Sort so they are removable in bottom-up # order. chdir($mirrordir) or die "chdir $mirrordir: $!"; - system("find . -depth -type d ! -name . ! -name .. -print0 | xargs -0 rmdir 2>/dev/null") if (! $do_dry_run); + system("find . -depth -type d ! -name . ! -name .. -print0 2>/dev/null | xargs -0 rmdir 2>/dev/null") if (! $do_dry_run); } if ($res != 0) { @@ -2287,7 +2287,7 @@ my $ignore; $ignore = "(".join("|", @ignores).")" if @ignores; # Remove all files in the mirror that we don't know about - foreach my $file (`find . -type f`) { + foreach my $file (`find . -type f 2>/dev/null`) { chomp $file; $file=~s:^\./::; next if (@di_dists && $file =~ m:installer-\w+/current/images/:); @@ -2430,11 +2430,11 @@ sub di_cleanup { # Clean up obsolete files - foreach my $image_dir (`find dists/ -type d -name images`) { + foreach my $image_dir (`find dists/ -type d -name images 2>/dev/null`) { next unless $image_dir =~ m:/installer-\w+/current/images$:; chomp $image_dir; chdir("$image_dir") or die "unable to chdir($image_dir): $!\n"; - foreach my $file (`find . -type f`) { + foreach my $file (`find . -type f 2>/dev/null`) { chomp $file; $file=~s:^\./::; if (! exists $di_files{$image_dir} || ! exists $di_files{$image_dir}{$file}) { @@ -2450,7 +2450,7 @@ # Clean up temporary D-I files (silently) if (-d "$tempdir/d-i") { chdir("$tempdir/d-i") or die "unable to chdir($tempdir/d-i): $!\n"; - foreach my $file (`find . -type f`) { + foreach my $file (`find . -type f 2>/dev/null`) { chomp $file; $file=~s:^\./::; unlink "$file" or die "unlink $tempdir/d-i/$file: $!\n";
Description: turn batch downloader into a function for later use. Author: Kees Cook <[email protected]> Bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=455082 Index: debmirror-2.4.4/debmirror =================================================================== --- debmirror-2.4.4.orig/debmirror 2010-04-05 11:41:00.392648967 -0700 +++ debmirror-2.4.4/debmirror 2010-04-05 11:41:11.083899315 -0700 @@ -1173,7 +1173,9 @@ # Download all package files that we need to get. say("Get package files."); -DOWNLOAD: { +batch_get(); + +sub batch_get { $_ = $download_method; # hftp (ftp using http mirror) method @@ -1194,7 +1196,7 @@ } } } - last DOWNLOAD; + return; }; # http method @@ -1215,7 +1217,7 @@ } } } - last DOWNLOAD; + return; }; # ftp method @@ -1235,7 +1237,7 @@ } } } - last DOWNLOAD; + return; }; # rsync method @@ -1316,7 +1318,7 @@ } } } - last DOWNLOAD; + return; }; }
Description: drop the needless section of rsync code for trailing list. Author: Kees Cook <[email protected]> Bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=455082 Index: debmirror-2.4.4/debmirror =================================================================== --- debmirror-2.4.4.orig/debmirror 2010-04-05 11:37:55.000000000 -0700 +++ debmirror-2.4.4/debmirror 2010-04-05 11:40:21.316399141 -0700 @@ -1247,12 +1247,16 @@ my @result; my $i=0; my $j=0; + my @tofetch; $opt = "$opt --progress" if $progress; $opt = "$opt -v" if $verbose; $opt = "$opt -v" if $debug; $opt = "$opt -n" if $do_dry_run; foreach my $file (sort keys %files) { - if (!$files{$file}) { + push(@tofetch, $file) if (!$files{$file}); + } + my $last = scalar(@tofetch); + foreach my $file (@tofetch) { my $dirname; my @dir; ($dirname) = $file =~ m:(.*/):; @@ -1261,7 +1265,10 @@ push (@result, "" . join('/', @dir[0..$_]) . "/"); } push (@result, "$file"); - if (++$j >= $rsync_batch) { + $i++; + $j++; + say("want $file ($i/$last $j/$rsync_batch)") if ($progress || $verbose); + if ($j >= $rsync_batch || $i == $last) { $j = 0; ($fh, $rsynctempfile) = tempfile(); if (@result) { @@ -1294,29 +1301,6 @@ $num_errors++; last; } - } - } - ($fh, $rsynctempfile) = tempfile(); - if (@result) { - @result = sort(@result); - my $prev = "not equal to $result[0]"; - @result = grep($_ ne $prev && ($prev = $_, 1), @result); - for (@result) { - print $fh "$_\n"; - } - system ("rsync --timeout=$timeout $opt $rsyncremote --include-from=$rsynctempfile --exclude='*' $mirrordir"); - close $fh; - foreach my $dest (@result) { - if (-f $dest) { - if (!check_lists($dest)) { - say("$dest failed md5sum check"); - $num_errors++; - } - } elsif (!-d $dest) { - say("$dest missing"); - $num_errors++; - } - } } return; };
Description: adjust logic to report on why a file is needed in verbose mode. Author: Kees Cook <[email protected]> Bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=455082 Index: debmirror-2.4.4/debmirror =================================================================== --- debmirror-2.4.4.orig/debmirror 2010-04-05 12:21:29.255149875 -0700 +++ debmirror-2.4.4/debmirror 2010-04-05 12:21:44.362648442 -0700 @@ -1436,19 +1436,29 @@ # It will return true if the tests show the file matches. sub check_file { my ($filename, $size, $md5sum)=...@_; - if (-f $filename and ($size == -s _ || $size == -1)) { + if (! -f $filename) { + say("Missing: $filename") if ($verbose); + return 0; + } + my $disksize = -s _; + if ($size == $disksize || $size == -1) { if ($check_md5sums || $size == -1) { open HANDLE, $filename or die "$filename: $!"; $md5->addfile(*HANDLE); my $digest = $md5->hexdigest; - return ($md5sum eq $digest); + return 1 if ($md5sum eq $digest); + say(sprintf("Mismatch '$filename': sum is %s, expected %s",$digest,$md5sum)) + if ($verbose); } else { # Assume it is ok, w/o md5 check. return 1; } } + elsif ($verbose) { + say(sprintf("Mismatch '$filename': size is %d, expected %d",$disksize,$size)); + } return 0; }
Description: actually check the return codes from rsync and other commands. Author: Kees Cook <[email protected]> Bug: http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=455082 Index: debmirror-2.4.4/debmirror =================================================================== --- debmirror-2.4.4.orig/debmirror 2010-04-05 11:40:30.292651025 -0700 +++ debmirror-2.4.4/debmirror 2010-04-05 11:40:35.402651381 -0700 @@ -1280,6 +1280,7 @@ } } system ("rsync --no-motd --timeout=$timeout $opt $rsyncremote --include-from=$rsynctempfile --exclude='*' $mirrordir"); + die "rsync failed!" if ($? != 0); close $fh; unlink $rsynctempfile; foreach my $dest (@result) { @@ -1781,7 +1782,10 @@ if (-f $tofile) { unlink($tofile) or die "unlink($tofile) failed: $!"; } - system("$command <$fromfile >$tofile"); + my $cmd="$command <$fromfile >$tofile"; + say("Running: $cmd") if ($verbose); + system("$cmd"); + die "Failed: $cmd\n" if ($? != 0); } sub split_dist {
