Author: ranger
Date: Thu Feb 15 20:17:44 2007
New Revision: 654
URL:
<http://svn.finkproject.org/websvn/listing.php?sc=1&rev=654&repname=user%3a+ranger>
Log:
clean up the find stuff, option for not pulling cvs again, more other misc
cleanups
Modified:
trunk/experimental/scripts/mirror.pl
Modified: trunk/experimental/scripts/mirror.pl
URL:
<http://svn.finkproject.org/websvn/diff.php?path=/trunk/experimental/scripts/mirror.pl&rev=654&repname=user%3a+ranger>
==============================================================================
--- trunk/experimental/scripts/mirror.pl (original)
+++ trunk/experimental/scripts/mirror.pl Thu Feb 15 20:17:44 2007
@@ -39,141 +39,170 @@
mkpath($WORKDIR . '/fink');
mkpath($DOWNLOADDIR);
chdir($WORKDIR . '/fink');
-run_ssh_command('checkout', 'dists');
-
-find( { wanted => \&find_fetch_infofile, follow => 0 }, $WORKDIR);
+
+if (not grep(/^--s/, @ARGV))
+{
+ print "- updating cvs\n";
+ run_ssh_command('checkout', 'dists');
+}
+
+print "- scanning info files\n";
+opendir(DIR, $WORKDIR . '/fink/dists') or die "unable to read from
$WORKDIR/fink/dists: $!";
+for my $dir (readdir(DIR))
+{
+ if ($dir =~ /^10/)
+ {
+ next if ($dir =~ /^10.2/);
+ print "searching $dir\n";
+ finddepth( { wanted => \&find_fetch_infofile, follow => 1 },
$WORKDIR . '/fink/dists/' . $dir);
+ }
+}
+closedir(DIR);
sub find_fetch_infofile
{
- return unless ( $File::Find::name =~ /\.info$/ );
- return if ( -d $File::Find::name );
- return if ( $File::Find::name =~ m#/CVS/# );
- return if ( $File::Find::name =~ m#/10.2(-gcc3.3)?/# );
-
- if ( $File::Find::name =~ m#/dists/([^/]+)/# )
- {
- my $dist = $1;
- $Fink::Config::config = Fink::Config->new_from_properties({
- basepath => $WORKDIR,
- distribution => $dist,
- });
- $Fink::Config::libpath = '/home/ranger/cvs/fink';
- }
- else
- {
- return;
- }
-
- for my $package ( Fink::PkgVersion->pkgversions_from_info_file(
$File::Find::name ) )
- {
- next if ( $package->get_license() =~ /Restrictive$/ );
- for my $suffix ($package->get_source_suffices)
+ my $shortname = $_;
+ my $dist;
+ print $File::Find::name, "\n";
+
+ if ( $File::Find::name =~ m#\.info$# and $File::Find::name !~ m#/CVS/#
and $File::Find::name !~ m#/10.2(-gcc3.3)?/#)
+ {
+ if ( $File::Find::name =~ m#/dists/([^/]+)/# )
{
- my $tarball = $package->get_tarball($suffix);
- my $checksums = {};
- $checksums->{'MD5'} = $package->param('Source' .
$suffix . '-MD5');
- my($checksum_type, $checksum) =
Fink::Checksum->parse_checksum($package->get_checksum($suffix));
- $checksums->{$checksum_type} = $checksum;
-
- if (not -l $DOWNLOADDIR . '/' . $tarball)
+ $dist = $1;
+ $Fink::Config::config =
Fink::Config->new_from_properties({
+ basepath => $WORKDIR,
+ distribution => $dist,
+ });
+ $Fink::Config::libpath = '/home/ranger/cvs/fink';
+
+ my ($tree) = $File::Find::name =~
m#/dists/[^/]+/([^/]+)/#;
+ print "- fetching files for $shortname ($dist/$tree)\n";
+ for my $package (
Fink::PkgVersion->pkgversions_from_info_file( $File::Find::name ) )
{
- my $file_checksums =
Fink::Checksum->get_all_checksums($DOWNLOADDIR . '/' . $tarball);
- my $master_checksum_type = 'MD5';
- if (exists
$file_checksums->{$master_checksum_type})
+ next if ( $package->get_license() =~
/Restrictive$/ );
+ for my $suffix ($package->get_source_suffices)
{
- mkpath($DOWNLOADDIR . '/md5/' .
$file_checksums->{$master_checksum_type});
- move($DOWNLOADDIR . '/' . $tarball,
$DOWNLOADDIR . '/md5/' . $file_checksums->{$master_checksum_type} . '/' .
$tarball);
- for my $checksum_type (keys
%$file_checksums)
- {
- next if ($key eq
$master_checksum_type);
- mkpath($DOWNLOADDIR . '/' .
lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}));
- symlink(
- '../../' .
lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type})
. '/' . $tarball,
- $DOWNLOADDIR . '/' .
lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}) . '/' .
$tarball
- );
- unlink( $DOWNLOADDIR . '/' .
$tarball );
- symlink(
-
lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type})
. '/' . $tarball,
- $DOWNLOADDIR . '/' .
$tarball,
- );
-
- }
- }
- }
-
- my $do_download = 0;
- for $checksum_type (keys %$checksums)
- {
- next if (not defined $checksum_type or
$checksum_type =~ /^\s*$/);
- my $check_file = $DOWNLOADDIR . '/' .
lc($checksum_type) . '/' . lc($checksums->{$checksum_type}) . '/' . $tarball;
- if (not -f $check_file)
- {
- print "$check_file does not exists,
downloading\n";
- $do_download = 1;
- }
- else
- {
- if (not
Fink::Checksum->validate($check_file, $checksums->{$checksum_type},
$checksum_type))
- {
- print "checksum on $check_file
does not match, downloading\n";
- $do_download = 1;
- }
- }
- }
- if ($do_download)
- {
- my $master_checksum_type = 'MD5';
- if (not exists $checksums->{'MD5'})
- {
- my @types = sort keys %$checksums;
- $master_checksum_type = shift(@types);
- }
- my $download_path = $DOWNLOADDIR . '/' .
lc($master_checksum_type) . '/' . $checksums->{$master_checksum_type};
- my $url = $package->get_source($suffix);
- print "url = $url, tarball = $tarball\n";
- my $returnval = &fetch_url_to_file({
- url => $url,
- filename => $tarball,
- custom_mirror =>
$package->get_custom_mirror($suffix),
- skip_master_mirror => 1,
- download_directory => $download_path,
- checksum =>
$checksums->{$master_checksum_type},
- checksum_type =>
$master_checksum_type,
- try_all_mirrors => 1,
- });
- if ($returnval != 0)
- {
- unlink($download_path . '/' . $tarball);
- warn "unable to download $url to
$download_path";
- next;
- }
-
- my $file_checksums =
Fink::Checksum->get_all_checksums($download_path . '/' . $tarball);
- for my $checksum_type (keys %$file_checksums)
- {
- if ($checksum_type eq
$master_checksum_type)
- {
- if
($file_checksums->{$checksum_type} ne $checksums->{$checksum_type})
- {
- warn "downloaded file
has a different checksum than expected ($file_checksums->{$checksum_type} ne
$checksums->{$checksum_type})";
+ my $tarball =
$package->get_tarball($suffix);
+ print " - $tarball... ";
+ my $checksums = {};
+ $checksums->{'MD5'} =
$package->param('Source' . $suffix . '-MD5');
+ my($checksum_type, $checksum) =
Fink::Checksum->parse_checksum($package->get_checksum($suffix));
+ $checksums->{$checksum_type} =
$checksum;
+
+ if (not -l $DOWNLOADDIR . '/' .
$tarball)
+ {
+ my $file_checksums =
Fink::Checksum->get_all_checksums($DOWNLOADDIR . '/' . $tarball);
+ my $master_checksum_type =
'MD5';
+ if (exists
$file_checksums->{$master_checksum_type})
+ {
+ mkpath($DOWNLOADDIR .
'/md5/' . $file_checksums->{$master_checksum_type});
+ move($DOWNLOADDIR . '/'
. $tarball, $DOWNLOADDIR . '/md5/' . $file_checksums->{$master_checksum_type} .
'/' . $tarball);
+ for my $checksum_type
(keys %$file_checksums)
+ {
+ next if ($key
eq $master_checksum_type);
+
mkpath($DOWNLOADDIR . '/' . lc($checksum_type) . '/' .
lc($file_checksums->{$checksum_type}));
+ symlink(
+
'../../' . lc($master_checksum_type) . '/' .
lc($file_checksums->{$master_checksum_type}) . '/' . $tarball,
+
$DOWNLOADDIR . '/' . lc($checksum_type) . '/' .
lc($file_checksums->{$checksum_type}) . '/' . $tarball
+ );
+ unlink(
$DOWNLOADDIR . '/' . $tarball );
+ symlink(
+
lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type})
. '/' . $tarball,
+
$DOWNLOADDIR . '/' . $tarball,
+ );
+
+ }
+ }
+ }
+
+ my $do_download = 0;
+ for $checksum_type (keys %$checksums)
+ {
+ next if (not defined
$checksum_type or $checksum_type =~ /^\s*$/);
+ my $check_file = $DOWNLOADDIR .
'/' . lc($checksum_type) . '/' . lc($checksums->{$checksum_type}) . '/' .
$tarball;
+ if (not -f $check_file)
+ {
+ print "$check_file does
not exists, downloading\n";
+ $do_download = 1;
+ }
+ else
+ {
+ if (not
Fink::Checksum->validate($check_file, $checksums->{$checksum_type},
$checksum_type))
+ {
+ print "checksum
on $check_file does not match, downloading\n";
+ $do_download =
1;
+ }
+ }
+ }
+ if ($do_download)
+ {
+ print "downloading\n";
+ my $master_checksum_type =
'MD5';
+ if (not exists
$checksums->{'MD5'})
+ {
+ my @types = sort keys
%$checksums;
+ $master_checksum_type =
shift(@types);
+ }
+ my $download_path =
$DOWNLOADDIR . '/' . lc($master_checksum_type) . '/' .
$checksums->{$master_checksum_type};
+ my $url =
$package->get_source($suffix);
+ my $returnval =
&fetch_url_to_file({
+ url =>
$url,
+ filename =>
$tarball,
+ custom_mirror =>
$package->get_custom_mirror($suffix),
+ skip_master_mirror => 1,
+ download_directory =>
$download_path,
+ checksum =>
$checksums->{$master_checksum_type},
+ checksum_type =>
$master_checksum_type,
+ try_all_mirrors => 1,
+ });
+ if ($returnval != 0)
+ {
+ unlink($download_path .
'/' . $tarball);
+ warn "unable to
download $url to $download_path";
+ next;
+ }
+
+ my $file_checksums =
Fink::Checksum->get_all_checksums($download_path . '/' . $tarball);
+ for my $checksum_type (keys
%$file_checksums)
+ {
+ if ($checksum_type eq
$master_checksum_type)
+ {
+ if
($file_checksums->{$checksum_type} ne $checksums->{$checksum_type})
+ {
+ warn
"downloaded file has a different checksum than expected
($file_checksums->{$checksum_type} ne $checksums->{$checksum_type})";
+ }
+ }
+ else
+ {
+
mkpath($DOWNLOADDIR . '/' . lc($checksum_type) . '/' .
lc($file_checksums->{$checksum_type}));
+ symlink(
+
'../../' . lc($master_checksum_type) . '/' .
lc($checksums->{$master_checksum_type}) . '/' . $tarball,
+
$DOWNLOADDIR . '/' . lc($checksum_type) . '/' .
lc($file_checksums->{$checksum_type}) . '/' . $tarball
+ );
+ unlink(
$DOWNLOADDIR . '/' . $tarball );
+ symlink(
+
lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/'
. $tarball,
+
$DOWNLOADDIR . '/' . $tarball,
+ );
+ }
}
}
else
{
- mkpath($DOWNLOADDIR . '/' .
lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}));
- symlink(
- '../../' .
lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/'
. $tarball,
- $DOWNLOADDIR . '/' .
lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}) . '/' .
$tarball
- );
- unlink( $DOWNLOADDIR . '/' .
$tarball );
- symlink(
-
lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/'
. $tarball,
- $DOWNLOADDIR . '/' .
$tarball,
- );
+ print "exists\n";
}
}
}
}
+ else
+ {
+ print "no dist? $File::Find::name\n";
+ }
+ }
+ else
+ {
+ print "failed match: $File::Find::name\n";
}
}
-------------------------------------------------------------------------
Take Surveys. Earn Cash. Influence the Future of IT
Join SourceForge.net's Techsay panel and you'll get the chance to share your
opinions on IT & business topics through brief surveys-and earn cash
http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV
_______________________________________________
Fink-commits mailing list
Fink-commits@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/fink-commits
