Author: eelco
Date: Mon Apr 11 13:16:54 2011
New Revision: 26781
URL: https://svn.nixos.org/websvn/nix/?rev=26781&sc=1
Log:
* Read manifests directly into the database, rather than first reading
them into memory. This brings memory use down to (more or less)
O(1). For instance, on my test case, the maximum resident size of
download-using-manifests while filling the DB went from 142 MiB to
11 MiB.
Modified:
nix/trunk/scripts/NixManifest.pm.in
Modified: nix/trunk/scripts/NixManifest.pm.in
==============================================================================
--- nix/trunk/scripts/NixManifest.pm.in Mon Apr 11 12:40:13 2011 (r26780)
+++ nix/trunk/scripts/NixManifest.pm.in Mon Apr 11 13:16:54 2011 (r26781)
@@ -6,6 +6,23 @@
use Fcntl ':flock';
+sub addNAR {
+ my ($narFiles, $storePath, $info) = @_;
+
+ $$narFiles{$storePath} = []
+ unless defined $$narFiles{$storePath};
+
+ my $narFileList = $$narFiles{$storePath};
+
+ my $found = 0;
+ foreach my $narFile (@{$narFileList}) {
+ $found = 1 if $narFile->{url} eq $info->{url};
+ }
+
+ push @{$narFileList}, $info if !$found;
+}
+
+
sub addPatch {
my ($patches, $storePath, $patch) = @_;
@@ -27,8 +44,8 @@
}
-sub readManifest {
- my ($manifest, $narFiles, $patches) = @_;
+sub readManifest_ {
+ my ($manifest, $addNAR, $addPatch) = @_;
open MANIFEST, "<$manifest"
or die "cannot open `$manifest': $!";
@@ -72,35 +89,22 @@
$inside = 0;
if ($type eq "narfile") {
-
- $$narFiles{$storePath} = []
- unless defined $$narFiles{$storePath};
-
- my $narFileList = $$narFiles{$storePath};
-
- my $found = 0;
- foreach my $narFile (@{$narFileList}) {
- $found = 1 if $narFile->{url} eq $url;
- }
- if (!$found) {
- push @{$narFileList},
- { url => $url, hash => $hash, size => $size
- , narHash => $narHash, narSize => $narSize
- , references => $references
- , deriver => $deriver
- , system => $system
- };
- }
-
+ &$addNAR($storePath,
+ { url => $url, hash => $hash, size => $size
+ , narHash => $narHash, narSize => $narSize
+ , references => $references
+ , deriver => $deriver
+ , system => $system
+ });
}
elsif ($type eq "patch") {
- addPatch $patches, $storePath,
+ &$addPatch($storePath,
{ url => $url, hash => $hash, size => $size
, basePath => $basePath, baseHash => $baseHash
, narHash => $narHash, narSize => $narSize
, patchType => $patchType
- };
+ });
}
}
@@ -134,6 +138,14 @@
}
+sub readManifest {
+ my ($manifest, $narFiles, $patches) = @_;
+ readManifest_($manifest,
+ sub { addNAR($narFiles, @_); },
+ sub { addPatch($patches, @_); } );
+}
+
+
sub writeManifest {
my ($manifest, $narFiles, $patches, $noCompress) = @_;
@@ -205,7 +217,7 @@
my $dbPath = "$manifestDir/cache.sqlite";
# Open/create the database.
- my $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
+ our $dbh = DBI->connect("dbi:SQLite:dbname=$dbPath", "", "")
or die "cannot open database `$dbPath'";
$dbh->{AutoCommit} = 0;
$dbh->{RaiseError} = 1;
@@ -279,47 +291,40 @@
"select 1 from Manifests where path = ? and timestamp = ?",
{}, $manifest, $timestamp)} == 1;
- # !!! Insert directly into the DB.
- my %narFiles;
- my %patches;
- my $version = readManifest($manifest, \%narFiles, \%patches);
-
- if ($version < 3) {
- die "you have an old-style manifest `$manifest'; please delete it";
- }
- if ($version >= 10) {
- die "manifest `$manifest' is too new; please delete it or upgrade
Nix";
- }
-
$dbh->do("delete from Manifests where path = ?", {}, $manifest);
$dbh->do("insert into Manifests(path, timestamp) values (?, ?)",
{}, $manifest, $timestamp);
- my $id = $dbh->sqlite_last_insert_rowid();
+ our $id = $dbh->sqlite_last_insert_rowid();
- foreach my $storePath (keys %narFiles) {
- my $narFileList = $narFiles{$storePath};
- foreach my $narFile (@{$narFiles{$storePath}}) {
- $dbh->do(
- "insert into NARs(manifest, storePath, url, hash, size,
narHash, " .
- "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?,
?, ?, ?, ?)",
- {}, $id, $storePath, $narFile->{url}, $narFile->{hash},
$narFile->{size},
- $narFile->{narHash}, $narFile->{narSize},
$narFile->{references},
- $narFile->{deriver}, $narFile->{system});
- }
- }
+ sub addNARToDB {
+ my ($storePath, $narFile) = @_;
+ $dbh->do(
+ "insert into NARs(manifest, storePath, url, hash, size,
narHash, " .
+ "narSize, refs, deriver, system) values (?, ?, ?, ?, ?, ?, ?,
?, ?, ?)",
+ {}, $id, $storePath, $narFile->{url}, $narFile->{hash},
$narFile->{size},
+ $narFile->{narHash}, $narFile->{narSize},
$narFile->{references},
+ $narFile->{deriver}, $narFile->{system});
+ };
+
+ sub addPatchToDB {
+ my ($storePath, $patch) = @_;
+ $dbh->do(
+ "insert into Patches(manifest, storePath, basePath, baseHash,
url, hash, " .
+ "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?, ?,
?, ?, ?, ?)",
+ {}, $id, $storePath, $patch->{basePath}, $patch->{baseHash},
$patch->{url},
+ $patch->{hash}, $patch->{size}, $patch->{narHash},
$patch->{narSize},
+ $patch->{patchType});
+ };
- foreach my $storePath (keys %patches) {
- my $patchList = $patches{$storePath};
- foreach my $patch (@{$patchList}) {
- $dbh->do(
- "insert into Patches(manifest, storePath, basePath,
baseHash, url, hash, " .
- "size, narHash, narSize, patchType) values (?, ?, ?, ?, ?,
?, ?, ?, ?, ?)",
- {}, $id, $storePath, $patch->{basePath},
$patch->{baseHash}, $patch->{url},
- $patch->{hash}, $patch->{size}, $patch->{narHash},
$patch->{narSize},
- $patch->{patchType});
- }
+ my $version = readManifest_($manifest, \&addNARToDB, \&addPatchToDB);
+
+ if ($version < 3) {
+ die "you have an old-style manifest `$manifest'; please delete it";
+ }
+ if ($version >= 10) {
+ die "manifest `$manifest' is too new; please delete it or upgrade
Nix";
}
}
_______________________________________________
nix-commits mailing list
[email protected]
http://mail.cs.uu.nl/mailman/listinfo/nix-commits
