Author: ranger Date: Thu Feb 15 20:17:44 2007 New Revision: 654 URL: <http://svn.finkproject.org/websvn/listing.php?sc=1&rev=654&repname=user%3a+ranger> Log: clean up the find stuff, option for not pulling cvs again, more other misc cleanups
Modified: trunk/experimental/scripts/mirror.pl Modified: trunk/experimental/scripts/mirror.pl URL: <http://svn.finkproject.org/websvn/diff.php?path=/trunk/experimental/scripts/mirror.pl&rev=654&repname=user%3a+ranger> ============================================================================== --- trunk/experimental/scripts/mirror.pl (original) +++ trunk/experimental/scripts/mirror.pl Thu Feb 15 20:17:44 2007 @@ -39,141 +39,170 @@ mkpath($WORKDIR . '/fink'); mkpath($DOWNLOADDIR); chdir($WORKDIR . '/fink'); -run_ssh_command('checkout', 'dists'); - -find( { wanted => \&find_fetch_infofile, follow => 0 }, $WORKDIR); + +if (not grep(/^--s/, @ARGV)) +{ + print "- updating cvs\n"; + run_ssh_command('checkout', 'dists'); +} + +print "- scanning info files\n"; +opendir(DIR, $WORKDIR . '/fink/dists') or die "unable to read from $WORKDIR/fink/dists: $!"; +for my $dir (readdir(DIR)) +{ + if ($dir =~ /^10/) + { + next if ($dir =~ /^10.2/); + print "searching $dir\n"; + finddepth( { wanted => \&find_fetch_infofile, follow => 1 }, $WORKDIR . '/fink/dists/' . $dir); + } +} +closedir(DIR); sub find_fetch_infofile { - return unless ( $File::Find::name =~ /\.info$/ ); - return if ( -d $File::Find::name ); - return if ( $File::Find::name =~ m#/CVS/# ); - return if ( $File::Find::name =~ m#/10.2(-gcc3.3)?/# ); - - if ( $File::Find::name =~ m#/dists/([^/]+)/# ) - { - my $dist = $1; - $Fink::Config::config = Fink::Config->new_from_properties({ - basepath => $WORKDIR, - distribution => $dist, - }); - $Fink::Config::libpath = '/home/ranger/cvs/fink'; - } - else - { - return; - } - - for my $package ( Fink::PkgVersion->pkgversions_from_info_file( $File::Find::name ) ) - { - next if ( $package->get_license() =~ /Restrictive$/ ); - for my $suffix ($package->get_source_suffices) + my $shortname = $_; + my $dist; + print $File::Find::name, "\n"; + + if ( $File::Find::name =~ m#\.info$# and $File::Find::name !~ m#/CVS/# and $File::Find::name !~ m#/10.2(-gcc3.3)?/#) + { + if ( $File::Find::name =~ m#/dists/([^/]+)/# ) { - my $tarball = $package->get_tarball($suffix); - my $checksums = {}; - $checksums->{'MD5'} = $package->param('Source' . $suffix . '-MD5'); - my($checksum_type, $checksum) = Fink::Checksum->parse_checksum($package->get_checksum($suffix)); - $checksums->{$checksum_type} = $checksum; - - if (not -l $DOWNLOADDIR . '/' . $tarball) + $dist = $1; + $Fink::Config::config = Fink::Config->new_from_properties({ + basepath => $WORKDIR, + distribution => $dist, + }); + $Fink::Config::libpath = '/home/ranger/cvs/fink'; + + my ($tree) = $File::Find::name =~ m#/dists/[^/]+/([^/]+)/#; + print "- fetching files for $shortname ($dist/$tree)\n"; + for my $package ( Fink::PkgVersion->pkgversions_from_info_file( $File::Find::name ) ) { - my $file_checksums = Fink::Checksum->get_all_checksums($DOWNLOADDIR . '/' . $tarball); - my $master_checksum_type = 'MD5'; - if (exists $file_checksums->{$master_checksum_type}) + next if ( $package->get_license() =~ /Restrictive$/ ); + for my $suffix ($package->get_source_suffices) { - mkpath($DOWNLOADDIR . '/md5/' . $file_checksums->{$master_checksum_type}); - move($DOWNLOADDIR . '/' . $tarball, $DOWNLOADDIR . '/md5/' . $file_checksums->{$master_checksum_type} . '/' . $tarball); - for my $checksum_type (keys %$file_checksums) - { - next if ($key eq $master_checksum_type); - mkpath($DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type})); - symlink( - '../../' . lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type}) . '/' . $tarball, - $DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}) . '/' . $tarball - ); - unlink( $DOWNLOADDIR . '/' . $tarball ); - symlink( - lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type}) . '/' . $tarball, - $DOWNLOADDIR . '/' . $tarball, - ); - - } - } - } - - my $do_download = 0; - for $checksum_type (keys %$checksums) - { - next if (not defined $checksum_type or $checksum_type =~ /^\s*$/); - my $check_file = $DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($checksums->{$checksum_type}) . '/' . $tarball; - if (not -f $check_file) - { - print "$check_file does not exists, downloading\n"; - $do_download = 1; - } - else - { - if (not Fink::Checksum->validate($check_file, $checksums->{$checksum_type}, $checksum_type)) - { - print "checksum on $check_file does not match, downloading\n"; - $do_download = 1; - } - } - } - if ($do_download) - { - my $master_checksum_type = 'MD5'; - if (not exists $checksums->{'MD5'}) - { - my @types = sort keys %$checksums; - $master_checksum_type = shift(@types); - } - my $download_path = $DOWNLOADDIR . '/' . lc($master_checksum_type) . '/' . $checksums->{$master_checksum_type}; - my $url = $package->get_source($suffix); - print "url = $url, tarball = $tarball\n"; - my $returnval = &fetch_url_to_file({ - url => $url, - filename => $tarball, - custom_mirror => $package->get_custom_mirror($suffix), - skip_master_mirror => 1, - download_directory => $download_path, - checksum => $checksums->{$master_checksum_type}, - checksum_type => $master_checksum_type, - try_all_mirrors => 1, - }); - if ($returnval != 0) - { - unlink($download_path . '/' . $tarball); - warn "unable to download $url to $download_path"; - next; - } - - my $file_checksums = Fink::Checksum->get_all_checksums($download_path . '/' . $tarball); - for my $checksum_type (keys %$file_checksums) - { - if ($checksum_type eq $master_checksum_type) - { - if ($file_checksums->{$checksum_type} ne $checksums->{$checksum_type}) - { - warn "downloaded file has a different checksum than expected ($file_checksums->{$checksum_type} ne $checksums->{$checksum_type})"; + my $tarball = $package->get_tarball($suffix); + print " - $tarball... "; + my $checksums = {}; + $checksums->{'MD5'} = $package->param('Source' . $suffix . '-MD5'); + my($checksum_type, $checksum) = Fink::Checksum->parse_checksum($package->get_checksum($suffix)); + $checksums->{$checksum_type} = $checksum; + + if (not -l $DOWNLOADDIR . '/' . $tarball) + { + my $file_checksums = Fink::Checksum->get_all_checksums($DOWNLOADDIR . '/' . $tarball); + my $master_checksum_type = 'MD5'; + if (exists $file_checksums->{$master_checksum_type}) + { + mkpath($DOWNLOADDIR . '/md5/' . $file_checksums->{$master_checksum_type}); + move($DOWNLOADDIR . '/' . $tarball, $DOWNLOADDIR . '/md5/' . $file_checksums->{$master_checksum_type} . '/' . $tarball); + for my $checksum_type (keys %$file_checksums) + { + next if ($key eq $master_checksum_type); + mkpath($DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type})); + symlink( + '../../' . lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type}) . '/' . $tarball, + $DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}) . '/' . $tarball + ); + unlink( $DOWNLOADDIR . '/' . $tarball ); + symlink( + lc($master_checksum_type) . '/' . lc($file_checksums->{$master_checksum_type}) . '/' . $tarball, + $DOWNLOADDIR . '/' . $tarball, + ); + + } + } + } + + my $do_download = 0; + for $checksum_type (keys %$checksums) + { + next if (not defined $checksum_type or $checksum_type =~ /^\s*$/); + my $check_file = $DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($checksums->{$checksum_type}) . '/' . $tarball; + if (not -f $check_file) + { + print "$check_file does not exists, downloading\n"; + $do_download = 1; + } + else + { + if (not Fink::Checksum->validate($check_file, $checksums->{$checksum_type}, $checksum_type)) + { + print "checksum on $check_file does not match, downloading\n"; + $do_download = 1; + } + } + } + if ($do_download) + { + print "downloading\n"; + my $master_checksum_type = 'MD5'; + if (not exists $checksums->{'MD5'}) + { + my @types = sort keys %$checksums; + $master_checksum_type = shift(@types); + } + my $download_path = $DOWNLOADDIR . '/' . lc($master_checksum_type) . '/' . $checksums->{$master_checksum_type}; + my $url = $package->get_source($suffix); + my $returnval = &fetch_url_to_file({ + url => $url, + filename => $tarball, + custom_mirror => $package->get_custom_mirror($suffix), + skip_master_mirror => 1, + download_directory => $download_path, + checksum => $checksums->{$master_checksum_type}, + checksum_type => $master_checksum_type, + try_all_mirrors => 1, + }); + if ($returnval != 0) + { + unlink($download_path . '/' . $tarball); + warn "unable to download $url to $download_path"; + next; + } + + my $file_checksums = Fink::Checksum->get_all_checksums($download_path . '/' . $tarball); + for my $checksum_type (keys %$file_checksums) + { + if ($checksum_type eq $master_checksum_type) + { + if ($file_checksums->{$checksum_type} ne $checksums->{$checksum_type}) + { + warn "downloaded file has a different checksum than expected ($file_checksums->{$checksum_type} ne $checksums->{$checksum_type})"; + } + } + else + { + mkpath($DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type})); + symlink( + '../../' . lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/' . $tarball, + $DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}) . '/' . $tarball + ); + unlink( $DOWNLOADDIR . '/' . $tarball ); + symlink( + lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/' . $tarball, + $DOWNLOADDIR . '/' . $tarball, + ); + } } } else { - mkpath($DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type})); - symlink( - '../../' . lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/' . $tarball, - $DOWNLOADDIR . '/' . lc($checksum_type) . '/' . lc($file_checksums->{$checksum_type}) . '/' . $tarball - ); - unlink( $DOWNLOADDIR . '/' . $tarball ); - symlink( - lc($master_checksum_type) . '/' . lc($checksums->{$master_checksum_type}) . '/' . $tarball, - $DOWNLOADDIR . '/' . $tarball, - ); + print "exists\n"; } } } } + else + { + print "no dist? $File::Find::name\n"; + } + } + else + { + print "failed match: $File::Find::name\n"; } } ------------------------------------------------------------------------- Take Surveys. Earn Cash. Influence the Future of IT Join SourceForge.net's Techsay panel and you'll get the chance to share your opinions on IT & business topics through brief surveys-and earn cash http://www.techsay.com/default.php?page=join.php&p=sourceforge&CID=DEVDEV _______________________________________________ Fink-commits mailing list Fink-commits@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/fink-commits