newer patch Re: pkg_add: extract out-of-order archives
Tweaked version. The previous version was working just fine, but this
one also marks hardlinks and symlinks as skippable, as should be.
It also adjusts the pace of progress slightly (making the meter jump AFTER
skipping the entry), and also it does prepare_to_extract systematically
(which will recheck links vs their packing-list entry for consistency)
I haven't seen any negative report from the previous version.
If this one does work too, I intend to commit it sometimes during the week.
Index: OpenBSD/Add.pm
===
RCS file: /build/data/openbsd/cvs/src/usr.sbin/pkg_add/OpenBSD/Add.pm,v
retrieving revision 1.153
diff -u -p -r1.153 Add.pm
--- OpenBSD/Add.pm 20 May 2014 05:55:43 - 1.153
+++ OpenBSD/Add.pm 13 Jun 2014 17:14:05 -
@@ -113,7 +113,34 @@ sub perform_extraction
$state-{partial} = $handle-{partial};
$state-{archive} = $handle-{location};
$state-{check_digest} = $handle-{plist}{check_digest};
- $state-progress-visit_with_size($handle-{plist}, 'extract', $state);
+ my ($wanted, $tied) = {};
+ $handle-{plist}-find_extractible($state, $wanted, $tied);
+ my $p = $state-progress-new_sizer($handle-{plist}, $state);
+ while (my $file = $state-{archive}-next) {
+ my $e = $tied-{$file-name};
+ if (defined $e) {
+ delete $tied-{$file-name};
+ $e-prepare_to_extract($state, $file);
+ $state-{archive}-skip;
+ $p-advance($e);
+ # skip to next;
+ }
+ $e = $wanted-{$file-name};
+ if (!defined $e) {
+ $state-Fatal(archive member not found #1,
+ $file-name);
+ }
+ delete $wanted-{$file-name};
+ $e-prepare_to_extract($state, $file);
+ $e-extract($state, $file);
+ $p-advance($e);
+ if (keys %$wanted == 0) {
+ return;
+ }
+ }
+ if (keys %$wanted 0) {
+ $state-fatal(Truncated archive);
+ }
}
my $user_tagged = {};
@@ -181,6 +208,10 @@ sub prepare_for_addition
{
}
+sub find_extractible
+{
+}
+
sub extract
{
my ($self, $state) = @_;
@@ -236,6 +267,16 @@ sub set_modes
}
}
+package OpenBSD::PackingElement::Meta;
+
+# XXX stuff that's invisible to find_extractible should be considered extracted
+# for the most part, otherwise we create broken partial packages
+sub find_extractible
+{
+ my ($self, $state, $wanted, $tied) = @_;
+ $state-{partial}{$self} = 1;
+}
+
package OpenBSD::PackingElement::ExtraInfo;
use OpenBSD::Error;
@@ -360,6 +401,18 @@ use File::Basename;
use File::Path;
use OpenBSD::Temp;
+sub find_extractible
+{
+ my ($self, $state, $wanted, $tied) = @_;
+ $wanted-{$self-name} = $self;
+ return;
+ if ($self-{tied} || $self-{link} || $self-{symlink}) {
+ $tied-{$self-name} = $self;
+ } else {
+ $wanted-{$self-name} = $self;
+ }
+}
+
sub prepare_for_addition
{
my ($self, $state, $pkgname) = @_;
@@ -384,19 +437,11 @@ sub prepare_for_addition
sub prepare_to_extract
{
- my ($self, $state) = @_;
+ my ($self, $state, $file) = @_;
my $fullname = $self-fullname;
my $destdir = $state-{destdir};
- my $file=$state-{archive}-next;
- if (!defined $file) {
- $state-fatal(truncated archive);
- }
$file-{cwd} = $self-cwd;
- if (!$file-check_name($self)) {
- $state-fatal(archive does not match #1 != #2,
- $file-name, $self-name);
- }
if (defined $self-{symlink} || $file-isSymLink) {
unless (defined $self-{symlink} $file-isSymLink) {
$state-fatal(bogus symlink #1, $self-name);
@@ -427,14 +472,11 @@ sub prepare_to_extract
if (defined $self-{symlink} $state-{do_faked}) {
$file-{linkname} = $destdir.$file-{linkname};
}
- return $file;
}
sub extract
{
- my ($self, $state) = @_;
-
- my $file = $self-prepare_to_extract($state);
+ my ($self, $state, $file) = @_;
if (defined $self-{link} || defined $self-{symlink}) {
$state-{archive}-skip;
@@ -483,6 +525,7 @@ sub extract
$state-say(extracting #1, $tempname) if $state-verbose = 3;
+
if (!$file-isFile) {
$state-fatal(can't extract #1, it's not a file,
$self-stringize);
@@ -558,6 +601,10 @@ sub prepare_for_addition
if ($s-avail 0) {
$s-report_overflow($state, $fname);
}
+}
+
+sub find_extractible
+{
}
sub extract
Index: OpenBSD/ProgressMeter.pm
pkg_add: extract out-of-order archives
I already sent a preliminary version of this patch to selected people.
Now that packages use long names when needed, we can pinpoint every file
in the archive based on its archived name.
This patch changes pkg_add to be able to extract archives that contain
elements out-of-order, e.g., match package contents to plist in whatever
order. Once the temporary elements are extracted, the actual extract
routine will put everything in place, run scripts, etc.
This is really important. Test/read/report failures to me.
Once this is in, there is another patch that will allow pkg_create to
create out-of-order archives.
The neat effect will be to speed up updates, by relegating unchanged files
to the end of the archive, so they don't need to be fetched at all.
the effect should be drastic on monsters like texlive, where most of
the files do not change at all from version to version.
Index: OpenBSD/Add.pm
===
RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/Add.pm,v
retrieving revision 1.153
diff -u -p -r1.153 Add.pm
--- OpenBSD/Add.pm 20 May 2014 05:55:43 - 1.153
+++ OpenBSD/Add.pm 12 Jun 2014 09:50:44 -
@@ -113,7 +113,33 @@ sub perform_extraction
$state-{partial} = $handle-{partial};
$state-{archive} = $handle-{location};
$state-{check_digest} = $handle-{plist}{check_digest};
- $state-progress-visit_with_size($handle-{plist}, 'extract', $state);
+ my ($wanted, $tied) = {};
+ $handle-{plist}-find_extractible($state, $wanted, $tied);
+ my $p = $state-progress-new_sizer($handle-{plist}, $state);
+ while (my $file = $state-{archive}-next) {
+ my $e = $tied-{$file-name};
+ if (defined $e) {
+ delete $tied-{$file-name};
+ $p-advance($e);
+ $state-{archive}-skip;
+ # skip to next;
+ }
+ $e = $wanted-{$file-name};
+ if (!defined $e) {
+ $state-Fatal(archive member not found #1,
+ $file-name);
+ }
+ delete $wanted-{$file-name};
+ $e-prepare_to_extract($state, $file);
+ $e-extract($state, $file);
+ $p-advance($e);
+ if (keys %$wanted == 0) {
+ return;
+ }
+ }
+ if (keys %$wanted 0) {
+ $state-fatal(Truncated archive);
+ }
}
my $user_tagged = {};
@@ -181,6 +207,10 @@ sub prepare_for_addition
{
}
+sub find_extractible
+{
+}
+
sub extract
{
my ($self, $state) = @_;
@@ -236,6 +266,16 @@ sub set_modes
}
}
+package OpenBSD::PackingElement::Meta;
+
+# XXX stuff that's invisible to find_extractible should be considered extracted
+# for the most part, otherwise we create broken partial packages
+sub find_extractible
+{
+ my ($self, $state, $wanted, $tied) = @_;
+ $state-{partial}{$self} = 1;
+}
+
package OpenBSD::PackingElement::ExtraInfo;
use OpenBSD::Error;
@@ -360,6 +400,18 @@ use File::Basename;
use File::Path;
use OpenBSD::Temp;
+sub find_extractible
+{
+ my ($self, $state, $wanted, $tied) = @_;
+ $wanted-{$self-name} = $self;
+ return;
+ if ($self-{tied}) {
+ $tied-{$self-name} = $self;
+ } else {
+ $wanted-{$self-name} = $self;
+ }
+}
+
sub prepare_for_addition
{
my ($self, $state, $pkgname) = @_;
@@ -384,19 +436,11 @@ sub prepare_for_addition
sub prepare_to_extract
{
- my ($self, $state) = @_;
+ my ($self, $state, $file) = @_;
my $fullname = $self-fullname;
my $destdir = $state-{destdir};
- my $file=$state-{archive}-next;
- if (!defined $file) {
- $state-fatal(truncated archive);
- }
$file-{cwd} = $self-cwd;
- if (!$file-check_name($self)) {
- $state-fatal(archive does not match #1 != #2,
- $file-name, $self-name);
- }
if (defined $self-{symlink} || $file-isSymLink) {
unless (defined $self-{symlink} $file-isSymLink) {
$state-fatal(bogus symlink #1, $self-name);
@@ -427,14 +471,11 @@ sub prepare_to_extract
if (defined $self-{symlink} $state-{do_faked}) {
$file-{linkname} = $destdir.$file-{linkname};
}
- return $file;
}
sub extract
{
- my ($self, $state) = @_;
-
- my $file = $self-prepare_to_extract($state);
+ my ($self, $state, $file) = @_;
if (defined $self-{link} || defined $self-{symlink}) {
$state-{archive}-skip;
@@ -483,6 +524,7 @@ sub extract
$state-say(extracting #1, $tempname) if $state-verbose = 3;
+
if (!$file-isFile) {
$state-fatal(can't extract #1, it's not a file,
