I already sent a preliminary version of this patch to selected people.
Now that packages use long names when needed, we can pinpoint every file
in the archive based on its archived name.
This patch changes pkg_add to be able to extract archives that contain
elements out-of-order, e.g., match package contents to plist in whatever
order. Once the temporary elements are extracted, the actual "extract"
routine will put everything in place, run scripts, etc.
This is really important. Test/read/report failures to me.
Once this is in, there is another patch that will allow pkg_create to
create out-of-order archives.
The neat effect will be to speed up updates, by relegating unchanged files
to the end of the archive, so they don't need to be fetched at all.
the effect should be drastic on monsters like texlive, where most of
the files do not change at all from version to version.
Index: OpenBSD/Add.pm
===================================================================
RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/Add.pm,v
retrieving revision 1.153
diff -u -p -r1.153 Add.pm
--- OpenBSD/Add.pm 20 May 2014 05:55:43 -0000 1.153
+++ OpenBSD/Add.pm 12 Jun 2014 09:50:44 -0000
@@ -113,7 +113,33 @@ sub perform_extraction
$state->{partial} = $handle->{partial};
$state->{archive} = $handle->{location};
$state->{check_digest} = $handle->{plist}{check_digest};
- $state->progress->visit_with_size($handle->{plist}, 'extract', $state);
+ my ($wanted, $tied) = {};
+ $handle->{plist}->find_extractible($state, $wanted, $tied);
+ my $p = $state->progress->new_sizer($handle->{plist}, $state);
+ while (my $file = $state->{archive}->next) {
+ my $e = $tied->{$file->name};
+ if (defined $e) {
+ delete $tied->{$file->name};
+ $p->advance($e);
+ $state->{archive}->skip;
+ # skip to next;
+ }
+ $e = $wanted->{$file->name};
+ if (!defined $e) {
+ $state->Fatal("archive member not found #1",
+ $file->name);
+ }
+ delete $wanted->{$file->name};
+ $e->prepare_to_extract($state, $file);
+ $e->extract($state, $file);
+ $p->advance($e);
+ if (keys %$wanted == 0) {
+ return;
+ }
+ }
+ if (keys %$wanted > 0) {
+ $state->fatal("Truncated archive");
+ }
}
my $user_tagged = {};
@@ -181,6 +207,10 @@ sub prepare_for_addition
{
}
+sub find_extractible
+{
+}
+
sub extract
{
my ($self, $state) = @_;
@@ -236,6 +266,16 @@ sub set_modes
}
}
+package OpenBSD::PackingElement::Meta;
+
+# XXX stuff that's invisible to find_extractible should be considered extracted
+# for the most part, otherwise we create broken partial packages
+sub find_extractible
+{
+ my ($self, $state, $wanted, $tied) = @_;
+ $state->{partial}{$self} = 1;
+}
+
package OpenBSD::PackingElement::ExtraInfo;
use OpenBSD::Error;
@@ -360,6 +400,18 @@ use File::Basename;
use File::Path;
use OpenBSD::Temp;
+sub find_extractible
+{
+ my ($self, $state, $wanted, $tied) = @_;
+ $wanted->{$self->name} = $self;
+ return;
+ if ($self->{tied}) {
+ $tied->{$self->name} = $self;
+ } else {
+ $wanted->{$self->name} = $self;
+ }
+}
+
sub prepare_for_addition
{
my ($self, $state, $pkgname) = @_;
@@ -384,19 +436,11 @@ sub prepare_for_addition
sub prepare_to_extract
{
- my ($self, $state) = @_;
+ my ($self, $state, $file) = @_;
my $fullname = $self->fullname;
my $destdir = $state->{destdir};
- my $file=$state->{archive}->next;
- if (!defined $file) {
- $state->fatal("truncated archive");
- }
$file->{cwd} = $self->cwd;
- if (!$file->check_name($self)) {
- $state->fatal("archive does not match #1 != #2",
- $file->name, $self->name);
- }
if (defined $self->{symlink} || $file->isSymLink) {
unless (defined $self->{symlink} && $file->isSymLink) {
$state->fatal("bogus symlink #1", $self->name);
@@ -427,14 +471,11 @@ sub prepare_to_extract
if (defined $self->{symlink} && $state->{do_faked}) {
$file->{linkname} = $destdir.$file->{linkname};
}
- return $file;
}
sub extract
{
- my ($self, $state) = @_;
-
- my $file = $self->prepare_to_extract($state);
+ my ($self, $state, $file) = @_;
if (defined $self->{link} || defined $self->{symlink}) {
$state->{archive}->skip;
@@ -483,6 +524,7 @@ sub extract
$state->say("extracting #1", $tempname) if $state->verbose >= 3;
+
if (!$file->isFile) {
$state->fatal("can't extract #1, it's not a file",
$self->stringize);
@@ -558,6 +600,10 @@ sub prepare_for_addition
if ($s->avail < 0) {
$s->report_overflow($state, $fname);
}
+}
+
+sub find_extractible
+{
}
sub extract
Index: OpenBSD/ProgressMeter.pm
===================================================================
RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/ProgressMeter.pm,v
retrieving revision 1.40
diff -u -p -r1.40 ProgressMeter.pm
--- OpenBSD/ProgressMeter.pm 23 Dec 2013 15:04:37 -0000 1.40
+++ OpenBSD/ProgressMeter.pm 12 Jun 2014 09:50:44 -0000
@@ -47,6 +47,15 @@ sub errprint
print STDERR @_;
}
+sub new_sizer
+{
+ return shift;
+}
+
+sub advance
+{
+}
+
sub for_list
{
my ($self, $msg, $l, $code) = @_;
Index: OpenBSD/ProgressMeter/Term.pm
===================================================================
RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/ProgressMeter/Term.pm,v
retrieving revision 1.23
diff -u -p -r1.23 Term.pm
--- OpenBSD/ProgressMeter/Term.pm 18 Mar 2014 18:53:29 -0000 1.23
+++ OpenBSD/ProgressMeter/Term.pm 12 Jun 2014 09:50:44 -0000
@@ -20,11 +20,8 @@ use warnings;
package OpenBSD::PackingElement;
sub size_and
{
- my ($self, $progress, $donesize, $totsize, $method, @r) = @_;
- if (defined $self->{size}) {
- $$donesize += $self->{size};
- $progress->show($$donesize, $totsize);
- }
+ my ($self, $p, $method, @r) = @_;
+ $p->advance($self);
$self->$method(@r);
}
@@ -80,18 +77,28 @@ sub compute_count
sub visit_with_size
{
my ($progress, $plist, $method, $state, @r) = @_;
+ my $p = $progress->new_sizer($plist, $state);
+ $plist->size_and($p, $method, $state, @r);
+}
+
+sub new_sizer
+{
+ my ($progress, $plist, $state) = @_;
$plist->{totsize} //= compute_size($plist);
- my $donesize = 0;
- my $totsize = $plist->{totsize};
- $progress->show($donesize, $totsize);
+ my $p = bless {
+ progress => $progress,
+ totsize => $plist->{totsize},
+ donesize => 0,
+ state => $state
+ }, "_SizeCounter";
+ $progress->show(0, $p->{totsize});
if (defined $state->{archive}) {
$state->{archive}{callback} = sub {
my $done = shift;
- $progress->show($donesize + $done, $totsize);
+ $progress->show($p->{donesize} + $done, $p->{totsize});
};
}
- $plist->size_and($progress, \$donesize, $totsize,
- $method, $state, @r);
+ return $p;
}
sub visit_with_count
@@ -304,6 +311,20 @@ sub next
$todo //= 'ok';
print "\r$self->{header}: $todo\n";
+}
+
+package _SizeCounter;
+sub new
+{
+}
+
+sub advance
+{
+ my ($self, $e) = @_;
+ if (defined $e->{size}) {
+ $self->{donesize} += $e->{size};
+ $self->{progress}->show($self->{donesize}, $self->{totsize});
+ }
}
1;
