Author: eelco
Date: Mon Nov 29 16:14:43 2010
New Revision: 24922
URL: https://svn.nixos.org/websvn/nix/?rev=24922&sc=1
Log:
* Enable asynchronous patch generation (i.e. it doesn't block
mirroring of the channel).
Modified:
release/trunk/channels/mirror-channel.pl
Modified: release/trunk/channels/mirror-channel.pl
==============================================================================
--- release/trunk/channels/mirror-channel.pl Mon Nov 29 16:03:36 2010
(r24921)
+++ release/trunk/channels/mirror-channel.pl Mon Nov 29 16:14:43 2010
(r24922)
@@ -1,9 +1,21 @@
+# This script mirrors a remote Nix channel in the local filesystem.
+# It downloads the remote manifest, then any NAR files that are not
+# already available in the target directory. If $ENABLE_PATCHES is
+# set, it also generates patches between the NAR files in the old
+# version of the manifest and the new version. Because this script
+# can take a long time to finish, it uses a lock to guard against
+# concurrent updates, allowing it to be run periodically from a cron
+# job.
+
use strict;
use readmanifest;
+use GeneratePatches;
use File::Basename;
use File::stat;
-use File::Temp qw/tempfile/;
+use File::Temp qw/tempfile tempdir/;
use Fcntl ':flock';
+use POSIX qw(strftime);
+
if (scalar @ARGV != 6 && scalar @ARGV != 7) {
print STDERR "Syntax: perl mirror-channel.pl <src-channel-url>
<dst-channel-dir> <nar-dir> <nar-url> <patches-dir> <patches-url>
[<nix-exprs-url>]\n";
@@ -19,15 +31,21 @@
my $patchesPath = $ARGV[4];
my $patchesURL = $ARGV[5];
my $nixexprsURL = $ARGV[6] || "$srcChannelURL/nixexprs.tar.bz2";
+my $enablePatches = defined $ENV{'ENABLE_PATCHES'} && -e
"$dstChannelPath/MANIFEST";
die "$dstChannelPath doesn't exist\n" unless -d $dstChannelPath;
die "$narPath doesn't exist\n" unless -d $narPath;
die "$patchesPath doesn't exist\n" unless -d $patchesPath;
+my $tmpDir = tempdir("nix-mirror-XXXXXXX", TMPDIR => 1, CLEANUP => 1);
+print STDERR "$tmpDir\n";
+
+
open LOCK, ">$dstChannelPath/.lock" or die;
flock LOCK, LOCK_EX;
-system("date");
+print STDERR "started mirroring at ", strftime("%a %b %e %H:%M:%S %Y",
localtime), "\n";
+
# Read the old manifest, if available.
my %narFilesOld;
@@ -42,16 +60,15 @@
$knownURLs{$_->{url}} = $_ foreach @{$files};
}
+
# Fetch the new manifest.
-my ($fh, $tmpManifest) = tempfile(UNLINK => 1);
-system("$curl '$srcChannelURL/MANIFEST' > $tmpManifest") == 0 or die;
+my $srcManifest = "$tmpDir/MANIFEST.src";
+system("$curl '$srcChannelURL/MANIFEST' > $srcManifest") == 0 or die;
-# Read the manifest.
-my %narFiles;
-my %localPaths;
-my %patches;
-readManifest($tmpManifest, \%narFiles, \%localPaths, \%patches);
+# Read the manifest.
+my (%narFiles, %localPaths, %patches);
+readManifest($srcManifest, \%narFiles, \%localPaths, \%patches);
%localPaths = ();
%patches = (); # not supported yet
@@ -59,9 +76,11 @@
my $size = scalar (keys %narFiles);
print "$size store paths in manifest\n";
+
# Protect against Hydra problems that leave the channel empty.
die "cowardly refusing to mirror an empty channel" if $size == 0;
+
# Download every file that we don't already have, and update every URL
# to point to the mirror. Also fill in the size and hash fields in
# the manifest in order to be compatible with Nix < 0.13.
@@ -110,28 +129,68 @@
}
}
-# Write the new manifest.
-writeManifest("$dstChannelPath/MANIFEST.tmp", \%narFiles, \%patches);
-# Generate patches.
-if (defined $ENV{'ENABLE_PATCHES'} && -e "$dstChannelPath/MANIFEST.tmp") {
- system("generate-patches.pl $narPath $patchesPath $patchesURL
$dstChannelPath/MANIFEST $dstChannelPath/MANIFEST.tmp") == 0 or die;
+# Read all the old patches and propagate the useful ones. We use the
+# file "all-patches" to keep track of all patches that have been
+# generated in the past, so that patches are not lost if (for
+# instance) a package temporarily disappears from the source channel,
+# or if multiple instances of this script are running concurrently.
+my (%dummy1, %dummy2, %allPatches);
+
+sub readAllPatches {
+ readManifest("$patchesPath/all-patches", \%dummy1, \%dummy2, \%allPatches)
+ if -f "$patchesPath/all-patches";
}
+readAllPatches;
+
+propagatePatches \%allPatches, \%narFiles, \%patches;
+propagatePatches \%patchesOld, \%narFiles, \%patches; # not really needed
+
+
+# Make the temporary manifest available.
+writeManifest("$dstChannelPath/MANIFEST.tmp", \%narFiles, \%patches);
+
rename("$dstChannelPath/MANIFEST.tmp", "$dstChannelPath/MANIFEST") or die;
rename("$dstChannelPath/MANIFEST.tmp.bz2", "$dstChannelPath/MANIFEST.bz2") or
die;
-# Mirror nixexprs.tar.bz2.
+
+# Mirror nixexprs.tar.bz2. This should really be done atomically with
updating the manifest.
my $tmpFile = "$dstChannelPath/.tmp.$$.nixexprs.tar.bz2";
system("$curl '$nixexprsURL' > $tmpFile") == 0 or die "cannot download
`$nixexprsURL'";
rename($tmpFile, "$dstChannelPath/nixexprs.tar.bz2") or die "cannot rename
$tmpFile";
-# Remove ".hash.*" files corresponding to NARs that have been removed.
-#foreach my $fn (glob "$narPath/.hash.*") {
-# my $fn2 = $fn;
-# $fn2 =~ s/\.hash\.//;
-# if (! -e "$fn2") {
-# print STDERR "removing hash $fn\n";
-# unlink "$fn";
-# }
-#}
+
+# Release the lock on the manifest to allow the manifest to be updated
+# by other runs of this script while we're generating patches.
+flock LOCK, LOCK_UN;
+
+
+if ($enablePatches) {
+
+ # Generate patches asynchronously. This can take a long time.
+ generatePatches(\%narFilesOld, \%narFiles, \%allPatches, \%patches,
+ $narPath, $patchesPath, $patchesURL, $tmpDir);
+
+ # Lock all-patches.
+ open PLOCK, ">$patchesPath/all-patches.lock" or die;
+ flock PLOCK, LOCK_EX;
+
+ # Update the list of all patches. We need to reread all-patches
+ # and merge in our new patches because the file may have changed
+ # in the meantime.
+ readAllPatches;
+ copyPatches \%patches, \%allPatches;
+ writeManifest("$patchesPath/all-patches", {}, \%allPatches, 0);
+
+ # Reacquire the manifest lock.
+ flock LOCK, LOCK_EX;
+
+ # Rewrite the manifest. We have to reread it and propagate all
+ # patches because it may have changed in the meantime.
+ readManifest("$dstChannelPath/MANIFEST", \%narFiles, \%localPaths,
\%patches);
+
+ propagatePatches \%allPatches, \%narFiles, \%patches;
+
+ writeManifest("$dstChannelPath/MANIFEST", \%narFiles, \%patches);
+}
_______________________________________________
nix-commits mailing list
[email protected]
http://mail.cs.uu.nl/mailman/listinfo/nix-commits
