Apparently, nobody cares about fat packages. Not surprisingly, killing that code simplifies a few things. Especially since the necessity of passing arch around was only due to the possibility of fat packages...
Index: package.5 =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/package.5,v retrieving revision 1.9 diff -u -p -r1.9 package.5 --- package.5 11 Jan 2010 10:16:20 -0000 1.9 +++ package.5 24 Aug 2011 10:35:28 -0000 @@ -29,7 +29,6 @@ and .Xr pkg_merge 1 , and are usually manipulated using .Xr pkg_add 1 , -.Xr pkg_merge 1 , .Xr pkg_mklocatedb 1 , or .Xr pkg_info 1 . @@ -57,9 +56,6 @@ All types of archive contents can be pre including files, directories, hardlinks, symlinks, fifos, block and character devices. .Pp -A special extension to the format, dubbed fat packages, is described in -the next section. -.Pp In order to allow just-in-time extraction, packages always begin with a table of contents, named .Pa +CONTENTS . @@ -74,8 +70,8 @@ See for annotation details. .Pp This table of contents is always followed by a few special files, some of -which are optional: the package description (+DESC), an installation script -(+INSTALL), a display message (+DISPLAY), etc. +which are optional: the package description (+DESC), +a display message (+DISPLAY), etc. .Pp The ustar format has some limitations with respect to file names. Accordingly, the package tools will replace very long names with @@ -103,38 +99,17 @@ Once the packing-list signature is check will be checksummed, resulting in a .Sq just-in-time signature checking. -.Sh FAT PACKAGES DESCRIPTION -The -.Xr pkg_merge 1 -command can create fat packages, which coalesce several normal packages in -a single ustar archive, by interleaving their contents. -.Pp -Other tools, such as -.Xr pkg_add 1 , -are aware of fat packages and can handle them transparently. .Pp -In a fat package, every item has a small prefix that identifies the -original package. -For instance, after merging two packages, the package will usually -begin with -.Pa a/+CONTENTS -and -.Pa b/+CONTENTS . -Individual items will then begin with -.Pa ab/file , -for a file common to both packages; -.Pa a/file -for a file belonging to the first package; -and -.Pa b/file -for a file belonging to the second package. +Fat packages were removed in +.Ox 5.1 , +since no practical application was found. .Sh SEE ALSO .Xr pkg_add 1 , .Xr pkg_create 1 , .Xr pkg_info 1 , -.Xr pkg_merge 1 , .Xr packages 7 , .Xr packages-specs 7 +.Sh HISTORY .Sh STANDARDS Packages are valid gzip'ed ustar archives that can be extracted using .Xr tar 1 . Index: pkg_merge =================================================================== RCS file: pkg_merge diff -N pkg_merge --- pkg_merge 28 Jul 2010 12:19:54 -0000 1.21 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,257 +0,0 @@ -#! /usr/bin/perl -# Copyright (c) 2005-2007 Marc Espie <es...@openbsd.org> -# $OpenBSD: pkg_merge,v 1.21 2010/07/28 12:19:54 espie Exp $ -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -use strict; -use warnings; - -use OpenBSD::PackageLocator; -use OpenBSD::PackageInfo; -use OpenBSD::PackingList; -use OpenBSD::Getopt; -use OpenBSD::Error; -use OpenBSD::Ustar; -use OpenBSD::ArcCheck; -use OpenBSD::Paths; -use OpenBSD::State; -use File::Copy; -use File::Path; - -package OpenBSD::PackingElement; -sub copy_over {} - -sub mark_tocopy {} - -sub make_alias {} -package OpenBSD::PackingElement::FileBase; -sub mark_tocopy -{ - my ($self, $list) = @_; - push(@$list, $self); -} - -sub copy_over -{ - my ($self, $wrarc, $prefix, $pkg, $state) = @_; - my $e = $pkg->{pkg}->next; - if (!$e->check_name($self)) { - $state->fatal("Names don't match: #1 vs #2", - $e->{name}, $self->{name}); - } - $e->{name} = $prefix."/".$e->{name}; - $e->copy_long($wrarc); -} - -sub make_alias -{ - my ($self, $wrarc, $prefix, $pkg, $alias, $state) = @_; - my $e = $pkg->{pkg}->next; - if (!$e->check_name($self)) { - $state->fatal("Names don't match: #1 vs #2", - $e->{name}, $self->{name}); - } - $e->{name} = $prefix."/".$e->{name}; - $e->alias($wrarc, "$prefix/$alias"); -} - -package OpenBSD::PackingElement::SpecialFile; -use File::Copy; - -sub mark_tocopy -{ - my ($self, $list) = @_; - push(@$list, $self); -} -sub copy_over -{ - my ($self, $wrarc, $prefix, $pkg, $state) = @_; - if (defined $wrarc) { - $wrarc->destdir($pkg->{dir}); - my $e = $wrarc->prepare($self->{name}); - $e->{name} = "$prefix/".$e->{name}; - $e->write; - } -} - - -package main; - -sub find_equal -{ - my $list = shift; - my $name = $list->[0]->{tocopy}->[0]->{name}; - for my $pkg (@$list) { - if ($pkg->{tocopy}->[0]->{name} ne $name) { - return undef; - } - } - return $name; -} - -sub occurs_first_or_not -{ - my ($name, $list) = @_; - for my $pkg (@$list) { - if ($pkg->{tocopy}->[0]->{name} eq $name) { - next; - } - for my $i (@{$pkg->{tocopy}}) { - if ($i->{name} eq $name) { - return 0; - } - } - } - return 1; -} - -sub occurs_first -{ - my $list= shift; - - for my $pkg (@$list) { - my $name = $pkg->{tocopy}->[0]->{name}; - if (occurs_first_or_not($name, $list)) { - return $name; - } - } - return undef; -} - - -my $ui = OpenBSD::State->new("pkg_merge"); -$ui->{no_exports} = 1; -$ui->handle_options('o:v', '[-v] -o result pkg pkg2 ...'); -my $out = $ui->opt('o'); -my $verbose = $ui->opt('v'); -if (!$out) { - $ui->usage("Missing -o result"); -} -if (-e $out) { - $ui->unlink($verbose, $out); -} - - -if (@ARGV < 2) { - $ui->usage("Can't merge less than two packages"); -} -my @tomerge; - -my $prefix = 'a'; -my $allprefix = ''; -open(my $outfh, "|-", OpenBSD::Paths->gzip, "-o", $out); - -my $wrarc = OpenBSD::Ustar->new($outfh, $ui, "."); -for my $pkgname (@ARGV) { - my $true_package = $ui->repo->find($pkgname); - $ui->fatal("No such package #1", $pkgname) unless $true_package; - my $dir = $true_package->info; - my $plist = OpenBSD::PackingList->fromfile($dir.CONTENTS); - - my $in = { - plist => $plist, - dir => $dir, - prefix => $prefix, - tocopy => [], - pkg => $true_package - }; - my $e = OpenBSD::PackingElement::FCONTENTS->new(CONTENTS); - $e->copy_over($wrarc, $prefix, $true_package, $ui); - $plist->mark_tocopy($in->{tocopy}); - push(@tomerge, $in); - $prefix++; -} - -my $total_files = 0; -my $total_size = 0; - -# For now, we assume packing-lists contain the same items. -while(1) { - # kill empty lists - my @n = (); - for my $pkg (@tomerge) { - if (@{$pkg->{tocopy}} > 0) { - push(@n, $pkg); - } - } - @tomerge = @n; - if (@tomerge == 0) { - last; - } - # determine which item we want to copy (by name) - my $name; - - # easiest case: same name all around. - $name = find_equal(\@tomerge); - - # second case: a name that occurs first in some lists, - # and not in the others - if (!defined $name) { - $name = occurs_first(\@tomerge); - } - - # else, try random - if (!defined $name) { - $name = $tomerge[0]->{tocopy}->[0]->{name}; - } - - my $allprefix=''; - my $ref; - my @mergeable = (); - # select the mergeable items - for my $pkg (@tomerge) { - if ($pkg->{tocopy}->[0]->{name} eq $name) { - push(@mergeable, $pkg); - } - } - - my $all_copies = 0; - while (@mergeable > 0) { - my $pkg = shift @mergeable; - my $ref = shift @{$pkg->{tocopy}}; - my $copies = 0; - my $currentprefix = $pkg->{prefix}; - my @todo = (); - my @merged = (); - for my $cmp (@mergeable) { - if (defined $ref->{d} && $cmp->{tocopy}->[0]->{d}->equals($ref->{d})) { - push(@merged, $cmp); - $currentprefix .= $cmp->{prefix}; - $copies++; - } else { - push(@todo, $cmp); - } - } - $total_files += $copies; - if (defined $ref->{size}) { - $total_size += $ref->{size} * $copies; - } - $ref->copy_over($wrarc, $currentprefix, $pkg); - @mergeable = @todo; - $all_copies += $copies; - for my $pkg2 (@merged) { - my $i = shift @{$pkg2->{tocopy}}; - $i->make_alias($wrarc, $currentprefix, $pkg2, $name, $ui); - } - } - if ($verbose) { - if ($all_copies) { - $ui->say("#1 (shared: #2)", $name, $all_copies); - } else { - $ui->say("#1", $name); - } - } -} -$wrarc->close; -$ui->say("Shared #1 files for #2 bytes", $total_files, $total_size) if $verbose; Index: pkg_merge.1 =================================================================== RCS file: pkg_merge.1 diff -N pkg_merge.1 --- pkg_merge.1 10 May 2010 09:17:55 -0000 1.7 +++ /dev/null 1 Jan 1970 00:00:00 -0000 @@ -1,108 +0,0 @@ -.\" $OpenBSD: pkg_merge.1,v 1.7 2010/05/10 09:17:55 espie Exp $ -.\" Copyright (c) 2005 Marc Espie <es...@openbsd.org> -.\" -.\" Permission to use, copy, modify, and distribute this software for any -.\" purpose with or without fee is hereby granted, provided that the above -.\" copyright notice and this permission notice appear in all copies. -.\" -.\" THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -.\" WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -.\" MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -.\" ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -.\" WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -.\" ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -.\" OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -.\" -.Dd $Mdocdate: May 10 2010 $ -.Dt PKG_MERGE 1 -.Os -.Sh NAME -.Nm pkg_merge -.Nd merge several packages into a fat package -.Sh SYNOPSIS -.Nm pkg_merge -.Op Fl v -.Fl o Ar filename -.Ar pkg1 pkg2 Op Ar ... -.Sh DESCRIPTION -The -.Nm -command is used to merge -independent packages -.Ar pkg1 , -.Ar pkg2 , -.Ar ... , -into a fat package -.Ar filename -that contains all the information necessary -to install all the packages. -.Pp -The resulting fat package will often be smaller -than the sum of the individual packages, as -identical files will be shared. -For instance, emacs packages for individual -architectures share most of the contents besides -the emacs binary. -Starting with -.Ox 3.8 , -.Xr pkg_add 1 -handles fat packages transparently. -.Xr pkg_add 1 -automatically selects the actual package contents to -install based on package name and architecture. -.Pp -The options are as follows: -.Bl -tag -width opt -.It Fl o Ar filename -Store the result in -.Ar filename . -.It Fl v -Process individual files verbosely, showing what files -get shared in the fat package. -.El -.Sh FILE FORMATS -An -.Ox -package is a tarball conforming to the ustar specification in -Single -.Ux . -Normal packages always start with a -.Pa +CONTENTS -file (packing-list). -.Pp -Fat packages start with a list of -.Pa a/+CONTENTS , -.Pa b/+CONTENTS , -.Pa ... -(packing-lists for the individual packages). -.Pp -.Xr pkg_add 1 -performs some minimal parsing on these packing-lists -and selects the correct package based on the architecture -and package name. -.Pp -Once the correct package is found, -.Xr pkg_add 1 -will only extract files matching the corresponding directory prefix. -.Pp -For instance, if -.Pa b/+CONTENTS -is selected, -.Xr pkg_add 1 -will extract files like -.Pa b/foo , -.Pa ab/foo2 , -and -.Pa abc/foo3 , -but not -.Pa a/foo4 -(and it will strip the prefix in the process). -.Pp -.Nm -uses some heuristics to try and share as many files as possible. -.Sh SEE ALSO -.Xr pkg_add 1 , -.Xr pkg_create 1 , -.Xr package 5 -.Sh AUTHORS -.An Marc Espie Index: OpenBSD/Handle.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/Handle.pm,v retrieving revision 1.28 diff -u -p -r1.28 Handle.pm --- OpenBSD/Handle.pm 12 Jul 2011 10:30:29 -0000 1.28 +++ OpenBSD/Handle.pm 24 Aug 2011 10:35:28 -0000 @@ -147,7 +147,7 @@ sub create_old my $self= $class->new; $self->{name} = $pkgname; - my $location = $state->repo->installed->find($pkgname, $state->{arch}); + my $location = $state->repo->installed->find($pkgname); if (defined $location) { $self->{location} = $location; } @@ -223,7 +223,7 @@ sub get_location my $name = $handle->{name}; - my $location = $state->repo->find($name, $state->{arch}); + my $location = $state->repo->find($name); if (!$location) { $state->print("#1", $state->deptree_header($name)); $handle->set_error(NOT_FOUND); Index: OpenBSD/PackageLocation.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/PackageLocation.pm,v retrieving revision 1.29 diff -u -p -r1.29 PackageLocation.pm --- OpenBSD/PackageLocation.pm 20 Mar 2011 08:17:49 -0000 1.29 +++ OpenBSD/PackageLocation.pm 24 Aug 2011 10:35:28 -0000 @@ -26,24 +26,14 @@ use OpenBSD::Error; sub new { - my ($class, $repository, $name, $arch) = @_; + my ($class, $repository, $name) = @_; my $self = { repository => $repository, name => $repository->canonicalize($name), state => $repository->{state} }; - if (defined $arch) { - $self->{arch} = $arch; - } bless $self, $class; return $self; } -sub set_arch -{ - my ($self, $arch) = @_; - - $self->{arch} = $arch; -} - sub url { my $self = shift; @@ -122,32 +112,6 @@ sub find_contents } } -sub find_fat_contents -{ - my $self = shift; - - while (my $e = $self->_next) { - unless ($e->{name} =~ m/^(.*)\/\+CONTENTS$/o) { - last; - } - my $prefix = $1; - my $contents = $e->contents; - require OpenBSD::PackingList; - - my $plist = OpenBSD::PackingList->fromfile(\$contents, - \&OpenBSD::PackingList::FatOnly); - if (defined $self->name) { - next if $plist->pkgname ne $self->name; - } - if ($plist->has('arch')) { - if ($plist->{arch}->check($self->{arch})) { - $self->{filter} = $prefix; - return $contents; - } - } - } -} - sub contents { my ($self, $extra) = @_; @@ -159,12 +123,10 @@ sub contents my $contents = $self->find_contents($extra); if ($contents) { $self->unput; - return $contents; } - return $self->find_fat_contents; + return $contents; } - $self->{contents} = $self->find_contents || - $self->find_fat_contents; + $self->{contents} = $self->find_contents; } return $self->{contents}; @@ -339,20 +301,7 @@ sub getNext { my $self = shift; - my $e = $self->{_archive}->next; - if (defined $self->{filter}) { - if ($e->{name} =~ m/^(.*?)\/(.*)$/o) { - my ($beg, $name) = ($1, $2); - if (index($beg, $self->{filter}) == -1) { - return $self->getNext; - } - $e->{name} = $name; - if ($e->isHardLink) { - $e->{linkname} =~ s/^(.*?)\///o; - } - } - } - return $e; + return $self->{_archive}->next; } sub skip Index: OpenBSD/PackageLocator.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/PackageLocator.pm,v retrieving revision 1.98 diff -u -p -r1.98 PackageLocator.pm --- OpenBSD/PackageLocator.pm 24 Dec 2010 09:04:14 -0000 1.98 +++ OpenBSD/PackageLocator.pm 24 Aug 2011 10:35:28 -0000 @@ -65,34 +65,34 @@ sub path_parse sub find { - my ($self, $_, $arch, $state) = @_; + my ($self, $_, $state) = @_; my $package; if (m/[\/\:]/o) { my ($repository, $pkgname) = $self->path_parse($_, $state); - $package = $repository->find($pkgname, $arch); + $package = $repository->find($pkgname); if (defined $package) { $self->default_path($state)->add($repository); } } else { - $package = $self->default_path($state)->find($_, $arch); + $package = $self->default_path($state)->find($_); } return $package; } sub grabPlist { - my ($self, $_, $arch, $code, $state) = @_; + my ($self, $_, $code, $state) = @_; my $plist; if (m/[\/\:]/o) { my ($repository, $pkgname) = $self->path_parse($_, $state); - $plist = $repository->grabPlist($pkgname, $arch, $code); + $plist = $repository->grabPlist($pkgname, $code); if (defined $plist) { $self->default_path($state)->add($repository); } } else { - $plist = $self->default_path($state)->grabPlist($_, $arch, $code); + $plist = $self->default_path($state)->grabPlist($_, $code); } return $plist; } Index: OpenBSD/PackageRepository.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/PackageRepository.pm,v retrieving revision 1.93 diff -u -p -r1.93 PackageRepository.pm --- OpenBSD/PackageRepository.pm 24 Dec 2010 09:04:14 -0000 1.93 +++ OpenBSD/PackageRepository.pm 24 Aug 2011 10:35:28 -0000 @@ -235,8 +237,8 @@ sub open sub find { - my ($repository, $name, $arch) = @_; - my $self = $repository->new_location($name, $arch); + my ($repository, $name) = @_; + my $self = $repository->new_location($name); if ($self->contents) { return $self; @@ -245,8 +247,8 @@ sub find sub grabPlist { - my ($repository, $name, $arch, $code) = @_; - my $self = $repository->new_location($name, $arch); + my ($repository, $name, $code) = @_; + my $self = $repository->new_location($name); return $self->grabPlist($code); } Index: OpenBSD/PackingList.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/PackingList.pm,v retrieving revision 1.114 diff -u -p -r1.114 PackingList.pm --- OpenBSD/PackingList.pm 16 Jun 2011 14:48:36 -0000 1.114 +++ OpenBSD/PackingList.pm 24 Aug 2011 10:35:28 -0000 @@ -291,21 +291,6 @@ sub UpdateInfoOnly } } -sub FatOnly -{ - my ($fh, $cont) = @_; - my $_; - while (<$fh>) { - # XXX optimization - if (m/^\@arch\b/o) { - &$cont($_); - return; - } - next unless m/^\@(?:name\b)/o; - &$cont($_); - } -} - sub ConflictOnly { my ($fh, $cont) = @_; Index: OpenBSD/PkgAdd.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/PkgAdd.pm,v retrieving revision 1.32 diff -u -p -r1.32 PkgAdd.pm --- OpenBSD/PkgAdd.pm 23 Aug 2011 10:32:27 -0000 1.32 +++ OpenBSD/PkgAdd.pm 24 Aug 2011 10:35:28 -0000 @@ -1065,7 +1065,7 @@ sub process_parameters if (OpenBSD::PackageName::is_stem($pkgname)) { $l = $state->updater->stem2location($inst, $pkgname, $state); } else { - $l = $inst->find($pkgname, $state->{arch}); + $l = $inst->find($pkgname); } if (!defined $l) { $state->say("Problem finding #1", $pkgname); Index: OpenBSD/PkgDelete.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/PkgDelete.pm,v retrieving revision 1.23 diff -u -p -r1.23 PkgDelete.pm --- OpenBSD/PkgDelete.pm 23 Aug 2011 10:32:27 -0000 1.23 +++ OpenBSD/PkgDelete.pm 24 Aug 2011 10:35:28 -0000 @@ -222,7 +222,7 @@ sub process_parameters $l = $state->stem2location($inst, $pkgname, $state); } else { - $l = $inst->find($pkgname, $state->{arch}); + $l = $inst->find($pkgname); } if (!defined $l) { $state->say("Problem finding #1", $pkgname); Index: OpenBSD/State.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/State.pm,v retrieving revision 1.20 diff -u -p -r1.20 State.pm --- OpenBSD/State.pm 24 Dec 2010 09:04:14 -0000 1.20 +++ OpenBSD/State.pm 24 Aug 2011 10:35:28 -0000 @@ -119,10 +119,10 @@ sub path_parse sub find { - my ($self, $pkg, $arch) = @_; + my ($self, $pkg) = @_; require OpenBSD::PackageLocator; - return OpenBSD::PackageLocator->find($pkg, $arch, $self->{state}); + return OpenBSD::PackageLocator->find($pkg, $self->{state}); } sub match_locations @@ -135,10 +135,10 @@ sub match_locations sub grabPlist { - my ($self, $url, $arch, $code) = @_; + my ($self, $url, $code) = @_; require OpenBSD::PackageLocator; - return OpenBSD::PackageLocator->grabPlist($url, $arch, $code, $self->{state}); + return OpenBSD::PackageLocator->grabPlist($url, $code, $self->{state}); } sub path Index: OpenBSD/Update.pm =================================================================== RCS file: /cvs/src/usr.sbin/pkg_add/OpenBSD/Update.pm,v retrieving revision 1.149 diff -u -p -r1.149 Update.pm --- OpenBSD/Update.pm 19 Jul 2011 20:04:23 -0000 1.149 +++ OpenBSD/Update.pm 24 Aug 2011 10:35:28 -0000 @@ -148,7 +148,6 @@ sub process_handle } my @l2 = (); for my $loc (@$l) { - $loc->set_arch($state->{arch}); if (!$loc) { next; }