Update of /cvsroot/fink/scripts/distfiles
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv16447
Added Files:
newmirror
Log Message:
Add the distfiles mirroring script.
This script runs on the master distfile server, and fetches the distfiles
needed for each branch of each tree.
--- NEW FILE: newmirror ---
#!/usr/bin/perl
use DBI;
$tmpdir = "/tmp/tmpdistfiles/";
$realdir = "/Volumes/src/distfiles/";
`mkdir -p $tmpdir`;
%filelist = ();
%errorlist = ();
system("/usr/bin/rsync -a --delete rsync://fink.opendarwin.org/finkinfo
/sw/fink/");
print "Now setting fink to 10.3/stable\n";
unlink "/sw/etc/fink.conf";
unlink "/sw/fink/dists";
symlink "/sw/etc/fink.10.3.stable.conf", "/sw/etc/fink.conf";
symlink "/sw/fink/10.3", "/sw/fink/dists";
$dsystem = "fink/10.3/stable";
addfink();
print "Now setting fink to 10.3/unstable\n";
unlink "/sw/etc/fink.conf";
symlink "/sw/etc/fink.10.3.unstable.conf", "/sw/etc/fink.conf";
symlink "/sw/fink/10.3", "/sw/fink/dists";
$dsystem = "fink/10.3/unstable";
addfink();
print "Now setting fink to 10.4-transitional/stable\n";
unlink "/sw/etc/fink.conf";
unlink "/sw/fink/dists";
symlink "/sw/etc/fink.10.4-transitional.stable.conf", "/sw/etc/fink.conf";
symlink "/sw/fink/10.4-transitional", "/sw/fink/dists";
$dsystem = "fink/10.4-transitional/stable";
addfink();
print "Now setting fink to 10.4-transitional/unstable\n";
unlink "/sw/etc/fink.conf";
symlink "/sw/etc/fink.10.4-transitional.unstable.conf", "/sw/etc/fink.conf";
symlink "/sw/fink/10.4-transitional", "/sw/fink/dists";
$dsystem = "fink/10.4-transitional/unstable";
addfink();
print "Now setting fink to 10.4/stable\n";
unlink "/sw/etc/fink.conf";
unlink "/sw/fink/dists";
symlink "/sw/etc/fink.10.4.stable.conf", "/sw/etc/fink.conf";
symlink "/sw/fink/10.4", "/sw/fink/dists";
$dsystem = "fink/10.4/stable";
addfink();
print "Now setting fink to 10.4/unstable\n";
unlink "/sw/etc/fink.conf";
symlink "/sw/etc/fink.10.4.unstable.conf", "/sw/etc/fink.conf";
symlink "/sw/fink/10.4", "/sw/fink/dists";
$dsystem = "fink/10.4/unstable";
addfink();
chdir $tmpdir;
$dbh = DBI->connect ("DBI:mysql:distfiles:localhost", "distfiles", "secret",
{RaiseError => 1});
foreach $file (keys %filelist) {
$existsdb = 0;
if( grep(/research.att.com/,$filelist{$file}{'urlcount'}{0}) ) {
print "ERROR: not mirroring $file from STUPID LICENSE SITE:
$filelist{$file}{'urlcount'}{0}\n";
next;
}
$md5 = $filelist{$file}{'md5'};
if( $md5 eq "-" ) {
next;
}
$sth = $dbh->prepare("SELECT * FROM distfiles WHERE name = '$file' AND
hashtype = 'md5' AND hash = '$md5'");
$sth->execute();
if( $sth->rows == 0 ) {
if( -l "$realdir/$file" ) {
unlink "$realdir/$file";
}
if( -e "$realdir/$file" ) {
my $realmd5 = `md5 -q $realdir/$file`;
chomp $realmd5;
print "ERROR: previously downloaded $file md5 does not
match current specification: $md5 vs. $realmd5. Conflicing system:
$filelist{$file}{'system'}.\n";
mkdir "$realdir/md5/$realmd5";
rename "$realdir/$file", "$realdir/md5/$realmd5/$file";
}
}
$sth->finish();
if( -e "$realdir/$file" ) {
next;
}
$fetcherrors = 0;
$md5errors = 0;
for( $i = 0; $i < $filelist{$file}{'urlcount'}; $i++ ) {
if(
grep(/distfiles.*\.opendarwin\.org/,$filelist{$file}{'url'}{$i}) ) {
next;
}
if( -e "$tmpdir/$file" ) {
last;
}
$finalurl = $filelist{$file}{'url'}{$i};
`curl -L -s -o $tmpdir/$file $filelist{$file}{'url'}{$i}`;
#$? >>= 8 if defined $? and $? >= 256;
if( $? == 1792 && grep(/ftp:/,$filelist{$file}{'url'}{$i}) ) {
`curl -L -s --disable-epsv -o $tmpdir/$file
$filelist{$file}{'url'}{$i}`;
}
if( $? != 0 ) {
print "Error fetching $file from
$filelist{$file}{'url'}{$i}: $?\n";
$fetcherrors++;
unlink("$tmpdir/$file");
next;
}
if( ! -e "$tmpdir/$file" ) {
print "Didn't get the $file from
$filelist{$file}{'url'}{$i}\n";
next;
}
$curmd5 = `/sbin/md5 -q $tmpdir/$file`;
chomp $curmd5;
if( $md5 ne $curmd5 ) {
print "Error with the md5 for $file from
$filelist{$file}{'url'}{$i}: $md5 vs. $curmd5\n";
$md5errors++;
unlink("$tmpdir/$file");
next;
}
}
if( -e "$tmpdir/$file" ) {
print "Success getting $file\n";
$system = $filelist{$file}{'system'};
$sth = $dbh->prepare("INSERT INTO distfiles VALUES ('$file',
'$finalurl', now(), 'md5', '$md5', '$system')");
$sth->execute();
$sth->finish();
# Move the file in place.
$tmp = `openssl sha1 "$tmpdir/$file"`;
$sha1 = $tmp;
$sha1 =~ s/.*\=\ //;
mkdir "$realdir/md5/$md5";
`mv $tmpdir/$file $realdir/md5/$md5/$file`;
symlink "md5/$md5/$file", "$realdir/$file";
symlink "../../md5/$md5/$file", "$realdir/sha1/$sha1/$file";
} else {
print "Error getting $file. Mail $filelist{$file}{'email'}
about the problem. system: $filelist{$file}{'system'} \n";
$errorlist{$file}{'md5errors'} = $md5errors;
$errorlist{$file}{'fetcherrors'} = $fetcherrors;
$errorlist{$file}{'email'} = $filelist{$file}{'email'};
}
}
open FILE, ">>/var/log/distfile.log" || warn "AAAAAAAH! no log file: $!\n";
$time = time();
foreach $file (keys %errorlist) {
print FILE "$time $file $errorlist{$file}{'email'}
$errorlist{$file}{'md5errors'} $errorlist{$file}{'fetcherrors'}\n";
}
close FILE;
sub addfink {
system("/sw/bin/fink index");
open LIST, "/sw/bin/fink fetch-all -i --dry-run 2>/dev/null |" || die "Can't
get list from fink: $!\n";
while( <LIST> ) {
chomp;
@line = split;
$file = $line[0];
$md5 = $line[1];
$mirrorcnt = 2;
$curmd5 = 0;
if ( $file eq "" ) {
next;
}
if ( $file eq "Information" ) {
next;
}
if ( $file eq "sudo" ) {
next;
}
if ( $file eq "WARNING:" ) {
next;
}
if ( $file eq "Reading" ) {
next;
}
if ( $file eq "Ignoring" ) {
next;
}
if( defined $filelist{$file} ) {
if ($filelist{$file}{'md5'} ne $md5) {
print "ERROR: $file appeared more than once with
different md5's ($dsystem vs. $filelist{$file}{'system'} /
$filelist{$file}{'md5'} vs $md5)\n";
}
next;
}
$l = $#line;
$filelist{$file}{'md5'} = $md5;
$urlcount = 0;
for( $i = 2; defined $line[$i]; $i++ ) {
if( !grep(/\:\/\//,$line[$i]) ) {
$filelist{$file}{'email'} .= $line[$i];
$filelist{$file}{'email'} .= " ";
next;
}
$filelist{$file}{'url'}{$urlcount} = $line[$i];
$urlcount++;
}
$filelist{$file}{'urlcount'} = $urlcount;
$filelist{$file}{'system'} = $dsystem;
}
close(LIST);
}
-------------------------------------------------------
Using Tomcat but need to do more? Need to support web services, security?
Get stuff done quickly with pre-integrated technology to make your job easier
Download IBM WebSphere Application Server v.1.0.1 based on Apache Geronimo
http://sel.as-us.falkag.net/sel?cmd=lnk&kid=120709&bid=263057&dat=121642
_______________________________________________
Fink-commits mailing list
Fink-commits@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/fink-commits
