Author: eelco
Date: Wed Nov 23 15:13:37 2011
New Revision: 30540
URL: https://nixos.org/websvn/nix/?rev=30540&sc=1
Log:
* Add an API function exportPaths() that provides the functionality of
‘nix-store --export’.
* Add a Perl module that provides the functionality of
‘nix-copy-closure --to’. This is used by build-remote.pl so it no
longer needs to start a separate nix-copy-closure process. Also, it
uses the Perl API to do the export, so it doesn't need to start a
separate nix-store process either. As a result, nix-copy-closure
and build-remote.pl should no longer fail on very large closures due
to an "Argument list too long" error. (Note that having very many
dependencies in a single derivation can still fail because the
environment can become too large. Can't be helped though.)
Added:
nix/trunk/perl/lib/Nix/CopyClosure.pm
Modified:
nix/trunk/perl/Makefile.am
nix/trunk/perl/lib/Nix/Store.pm
nix/trunk/perl/lib/Nix/Store.xs
nix/trunk/scripts/build-remote.pl.in
nix/trunk/scripts/nix-copy-closure.in
nix/trunk/src/libstore/local-store.cc
nix/trunk/src/libstore/store-api.cc
nix/trunk/src/libstore/store-api.hh
nix/trunk/src/nix-store/nix-store.cc
Modified: nix/trunk/perl/Makefile.am
==============================================================================
--- nix/trunk/perl/Makefile.am Wed Nov 23 12:21:35 2011 (r30539)
+++ nix/trunk/perl/Makefile.am Wed Nov 23 15:13:37 2011 (r30540)
@@ -2,7 +2,7 @@
perlarchname := $(shell perl -e 'use Config; print $$Config{archname};')
perllibdir = $(libdir)/perl5/site_perl/$(perlversion)/$(perlarchname)
-PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm
lib/Nix/SSH.pm lib/Nix/Config.pm.in
+PERL_MODULES = lib/Nix/Store.pm lib/Nix/Manifest.pm lib/Nix/GeneratePatches.pm
lib/Nix/SSH.pm lib/Nix/CopyClosure.pm lib/Nix/Config.pm.in
all: $(PERL_MODULES:.in=)
ln -sfn $(abs_builddir)/.libs/libNixStore.so lib/Store.so
Added: nix/trunk/perl/lib/Nix/CopyClosure.pm
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ nix/trunk/perl/lib/Nix/CopyClosure.pm Wed Nov 23 15:13:37 2011
(r30540)
@@ -0,0 +1,46 @@
+package Nix::CopyClosure;
+
+use strict;
+use Nix::Config;
+use Nix::Store;
+
+
+sub copyTo {
+ my ($sshHost, $sshOpts, $storePaths, $compressor, $decompressor,
$includeOutputs, $dryRun, $sign) = @_;
+
+ $compressor = "$compressor |" if $compressor ne "";
+ $decompressor = "$decompressor |" if $decompressor ne "";
+
+ # Get the closure of this path.
+ my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs,
+ map { followLinksToStorePath $_ } @{$storePaths})));
+
+ # Ask the remote host which paths are invalid. Because of limits
+ # to the command line length, do this in chunks. Eventually,
+ # we'll want to use ‘--from-stdin’, but we can't rely on the
+ # target having this option yet.
+ my @missing = ();
+ while (scalar(@closure) > 0) {
+ my @ps = splice(@closure, 0, 1500);
+ open(READ, "set -f; ssh $sshHost @{$sshOpts} nix-store
--check-validity --print-invalid @ps|");
+ while (<READ>) {
+ chomp;
+ push @missing, $_;
+ }
+ close READ or die;
+ }
+
+ # Export the store paths and import them on the remote machine.
+ if (scalar @missing > 0) {
+ print STDERR "copying ", scalar @missing, " missing paths to
‘$sshHost’...\n";
+ #print STDERR " $_\n" foreach @missing;
+ unless ($dryRun) {
+ open SSH, "| $compressor ssh $sshHost @{$sshOpts} '$decompressor
nix-store --import'" or die;
+ exportPaths(fileno(SSH), $sign, @missing);
+ close SSH or die "copying store paths to remote machine `$sshHost'
failed: $?";
+ }
+ }
+}
+
+
+1;
Modified: nix/trunk/perl/lib/Nix/Store.pm
==============================================================================
--- nix/trunk/perl/lib/Nix/Store.pm Wed Nov 23 12:21:35 2011 (r30539)
+++ nix/trunk/perl/lib/Nix/Store.pm Wed Nov 23 15:13:37 2011 (r30540)
@@ -12,7 +12,7 @@
our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } );
-our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure
followLinksToStorePath);
+our @EXPORT = qw(isValidPath topoSortPaths computeFSClosure
followLinksToStorePath exportPaths);
our $VERSION = '0.15';
Modified: nix/trunk/perl/lib/Nix/Store.xs
==============================================================================
--- nix/trunk/perl/lib/Nix/Store.xs Wed Nov 23 12:21:35 2011 (r30539)
+++ nix/trunk/perl/lib/Nix/Store.xs Wed Nov 23 15:13:37 2011 (r30540)
@@ -146,3 +146,16 @@
}
OUTPUT:
RETVAL
+
+
+void exportPaths(int fd, int sign, ...)
+ PPCODE:
+ try {
+ doInit();
+ Paths paths;
+ for (int n = 2; n < items; ++n) paths.push_back(SvPV_nolen(ST(n)));
+ FdSink sink(fd);
+ exportPaths(*store, paths, sign, sink);
+ } catch (Error & e) {
+ croak(e.what());
+ }
Modified: nix/trunk/scripts/build-remote.pl.in
==============================================================================
--- nix/trunk/scripts/build-remote.pl.in Wed Nov 23 12:21:35 2011
(r30539)
+++ nix/trunk/scripts/build-remote.pl.in Wed Nov 23 15:13:37 2011
(r30540)
@@ -5,6 +5,7 @@
use IO::Handle;
use Nix::Config;
use Nix::SSH qw/sshOpts openSSHConnection/;
+use Nix::CopyClosure;
no warnings('once');
@@ -225,8 +226,7 @@
# Copy the derivation and its dependencies to the build machine.
-system("NIX_SSHOPTS=\"@sshOpts\" @bindir@/nix-copy-closure $hostName
$maybeSign $drvPath @inputs") == 0
- or die "cannot copy inputs to $hostName: $?";
+Nix::CopyClosure::copyTo($hostName, [ @sshOpts ], [ $drvPath, @inputs ], "",
"", 0, 0, $maybeSign ne "");
# Perform the build.
Modified: nix/trunk/scripts/nix-copy-closure.in
==============================================================================
--- nix/trunk/scripts/nix-copy-closure.in Wed Nov 23 12:21:35 2011
(r30539)
+++ nix/trunk/scripts/nix-copy-closure.in Wed Nov 23 15:13:37 2011
(r30540)
@@ -3,6 +3,7 @@
use Nix::SSH;
use Nix::Config;
use Nix::Store;
+use Nix::CopyClosure;
if (scalar @ARGV < 1) {
@@ -39,8 +40,8 @@
$sign = 1;
}
elsif ($arg eq "--gzip") {
- $compressor = "| gzip";
- $decompressor = "gunzip |";
+ $compressor = "gzip";
+ $decompressor = "gunzip";
}
elsif ($arg eq "--from") {
$toMode = 0;
@@ -67,30 +68,7 @@
if ($toMode) { # Copy TO the remote machine.
-
- # Get the closure of this path.
- my @allStorePaths = reverse(topoSortPaths(computeFSClosure(0,
$includeOutputs, map { followLinksToStorePath $_ } @storePaths)));
-
- # Ask the remote host which paths are invalid.
- open(READ, "set -f; ssh $sshHost @sshOpts nix-store --check-validity
--print-invalid @allStorePaths|");
- my @missing = ();
- while (<READ>) {
- chomp;
- push @missing, $_;
- }
- close READ or die;
-
- # Export the store paths and import them on the remote machine.
- if (scalar @missing > 0) {
- print STDERR "copying these missing paths:\n";
- print STDERR " $_\n" foreach @missing;
- unless ($dryRun) {
- my $extraOpts = $sign ? "--sign" : "";
- system("set -f; nix-store --export $extraOpts @missing $compressor
| ssh $sshHost @sshOpts '$decompressor nix-store --import'") == 0
- or die "copying store paths to remote machine `$sshHost'
failed: $?";
- }
- }
-
+ Nix::CopyClosure::copyTo($sshHost, [ @sshOpts ], [ @storePaths ],
$compressor, $decompressor, $includeOutputs, $dryRun, $sign);
}
else { # Copy FROM the remote machine.
@@ -112,8 +90,10 @@
# Export the store paths on the remote machine and import them on locally.
if (scalar @missing > 0) {
- print STDERR "copying these missing paths:\n";
- print STDERR " $_\n" foreach @missing;
+ print STDERR "copying ", scalar @missing, " missing paths from
‘$sshHost’...\n";
+ #print STDERR " $_\n" foreach @missing;
+ $compressor = "| $compressor" if $compressor ne "";
+ $decompressor = "$decompressor |" if $decompressor ne "";
unless ($dryRun) {
my $extraOpts = $sign ? "--sign" : "";
system("set -f; ssh $sshHost @sshOpts 'nix-store --export
$extraOpts @missing $compressor' | $decompressor $Nix::Config::binDir/nix-store
--import") == 0
Modified: nix/trunk/src/libstore/local-store.cc
==============================================================================
--- nix/trunk/src/libstore/local-store.cc Wed Nov 23 12:21:35 2011
(r30539)
+++ nix/trunk/src/libstore/local-store.cc Wed Nov 23 15:13:37 2011
(r30540)
@@ -1255,7 +1255,7 @@
bool haveSignature = readInt(hashAndReadSource) == 1;
if (requireSignature && !haveSignature)
- throw Error("imported archive lacks a signature");
+ throw Error(format("imported archive of `%1%' lacks a signature") %
dstPath);
if (haveSignature) {
string signature = readString(hashAndReadSource);
Modified: nix/trunk/src/libstore/store-api.cc
==============================================================================
--- nix/trunk/src/libstore/store-api.cc Wed Nov 23 12:21:35 2011 (r30539)
+++ nix/trunk/src/libstore/store-api.cc Wed Nov 23 15:13:37 2011 (r30540)
@@ -298,6 +298,17 @@
}
+void exportPaths(StoreAPI & store, const Paths & paths,
+ bool sign, Sink & sink)
+{
+ foreach (Paths::const_iterator, i, paths) {
+ writeInt(1, sink);
+ store.exportPath(*i, sign, sink);
+ }
+ writeInt(0, sink);
+}
+
+
}
Modified: nix/trunk/src/libstore/store-api.hh
==============================================================================
--- nix/trunk/src/libstore/store-api.hh Wed Nov 23 12:21:35 2011 (r30539)
+++ nix/trunk/src/libstore/store-api.hh Wed Nov 23 15:13:37 2011 (r30540)
@@ -342,6 +342,12 @@
bool hashGiven = false);
+/* Export multiple paths in the format expected by ‘nix-store
+ --import’. */
+void exportPaths(StoreAPI & store, const Paths & paths,
+ bool sign, Sink & sink);
+
+
}
Modified: nix/trunk/src/nix-store/nix-store.cc
==============================================================================
--- nix/trunk/src/nix-store/nix-store.cc Wed Nov 23 12:21:35 2011
(r30539)
+++ nix/trunk/src/nix-store/nix-store.cc Wed Nov 23 15:13:37 2011
(r30540)
@@ -594,11 +594,7 @@
else throw UsageError(format("unknown flag `%1%'") % *i);
FdSink sink(STDOUT_FILENO);
- for (Strings::iterator i = opArgs.begin(); i != opArgs.end(); ++i) {
- writeInt(1, sink);
- store->exportPath(*i, sign, sink);
- }
- writeInt(0, sink);
+ exportPaths(*store, opArgs, sign, sink);
}
_______________________________________________
nix-commits mailing list
[email protected]
http://lists.science.uu.nl/mailman/listinfo/nix-commits
