Hello community, here is the log from the commit of package build for openSUSE:Factory checked in at 2018-07-07 21:52:27 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/build (Old) and /work/SRC/openSUSE:Factory/.build.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "build" Sat Jul 7 21:52:27 2018 rev:115 rq:619816 version:20180629 Changes: -------- --- /work/SRC/openSUSE:Factory/build/build.changes 2018-04-06 17:41:08.745698941 +0200 +++ /work/SRC/openSUSE:Factory/.build.new/build.changes 2018-07-07 21:52:34.831761173 +0200 @@ -1,0 +2,34 @@ +Fri Jun 29 12:02:10 UTC 2018 - [email protected] + +- docker: + * use a correct prefixed label for disturl + * Support building multiple image types containing a docker image + * Set build user to root for docker/fissile builds + * Compress containers so that old kiwi versions can handle them + * Put obs-docker-support uninstall before the last user line + * Also pass arch and buildflavor when creating the containerinfo + * set disturl as container LABEL + * Uncompress kiwi generated containers + * build-vm-docker: type=bind instead of type=volume in --mount option +- computeblocklists/extractbuild: + * Tweak verbose logging of directories in computeblocklists/extractbuild + * add support for FIEMAP ioctl + * don't use bare F for file + * document and use alternate ioctl numbers + * more tweaks and sanity checks + * Fallback to encode/decode helpers if perl does not support 'Q' in pack/unpack + * Use 'eval' when querying the blocklist +- Misc: + * fix escaping in regexp + * log toplevel KIWI directories in verbose mode + * Split kiwi product parser in a separate function + * Always add extra dependencies when doing sysdeps expansion + * Fix REPOTYPE setting if the PKGCONFIG file is generated by a service + * support zypper's handling of versioned deps + * pam-modules got dropped on sles 15 + * testing for build specific filesystem, which are more important then worker defaults + * mark lint option as obsolete + * Make sure that we know all pre/vminstalls + + +------------------------------------------------------------------- Old: ---- obs-build-20180329.tar.gz New: ---- obs-build-20180629.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ build.spec ++++++ --- /var/tmp/diff_new_pack.QLoCvx/_old 2018-07-07 21:52:35.747760006 +0200 +++ /var/tmp/diff_new_pack.QLoCvx/_new 2018-07-07 21:52:35.747760006 +0200 @@ -22,7 +22,7 @@ Summary: A Script to Build SUSE Linux RPMs License: GPL-2.0-or-later AND GPL-2.0-only Group: Development/Tools/Building -Version: 20180329 +Version: 20180629 Release: 0 Source: obs-build-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-build ++++++ PKGBUILD ++++++ --- /var/tmp/diff_new_pack.QLoCvx/_old 2018-07-07 21:52:35.771759976 +0200 +++ /var/tmp/diff_new_pack.QLoCvx/_new 2018-07-07 21:52:35.775759970 +0200 @@ -1,5 +1,5 @@ pkgname=build -pkgver=20180329 +pkgver=20180629 pkgrel=0 pkgdesc="Build packages in sandbox" arch=('i686' 'x86_64') ++++++ build.dsc ++++++ --- /var/tmp/diff_new_pack.QLoCvx/_old 2018-07-07 21:52:35.795759945 +0200 +++ /var/tmp/diff_new_pack.QLoCvx/_new 2018-07-07 21:52:35.795759945 +0200 @@ -1,6 +1,6 @@ Format: 1.0 Source: build -Version: 20180329 +Version: 20180629 Binary: build Maintainer: Adrian Schroeter <[email protected]> Architecture: all ++++++ debian.changelog ++++++ --- /var/tmp/diff_new_pack.QLoCvx/_old 2018-07-07 21:52:35.815759920 +0200 +++ /var/tmp/diff_new_pack.QLoCvx/_new 2018-07-07 21:52:35.815759920 +0200 @@ -1,4 +1,4 @@ -build (20180329) unstable; urgency=low +build (20180629) unstable; urgency=low * Update to current git trunk - add sles11sp2 build config and adapt autodetection ++++++ obs-build-20180329.tar.gz -> obs-build-20180629.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/Build/Docker.pm new/obs-build-20180629/Build/Docker.pm --- old/obs-build-20180329/Build/Docker.pm 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/Build/Docker.pm 2018-06-29 07:58:19.000000000 +0200 @@ -77,7 +77,9 @@ shift @args if $args[0] =~ /^--(?:from|repo|type)$/ || $args[0] =~ /^-[tr]$/; shift @args; } - push @{$ret->{'deps'}}, grep {/^[a-zA-Z_0-9]/} @args; + my @deps = grep {/^[a-zA-Z_0-9]/} @args; + s/^([^<=>]+)([<=>]+)/$1 $2 / for @deps; + push @{$ret->{'deps'}}, @deps; } elsif ($args[0] eq 'ar' || $args[0] eq 'addrepo') { shift @args; while (@args && $args[0] =~ /^-/) { @@ -230,11 +232,13 @@ push @repos, "{ \"url\": \"$annorepo->{'url'}\" }"; } } + my $buildtime = time(); print "{\n"; print " \"tags\": [ ".join(', ', @tags)." ]"; print ",\n \"repos\": [ ".join(', ', @repos)." ]" if @repos; print ",\n \"file\": \"$image\"" if defined $image; print ",\n \"disturl\": \"$disturl\"" if defined $disturl; + print ",\n \"buildtime\": $buildtime"; print "\n}\n"; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/Build/Kiwi.pm new/obs-build-20180629/Build/Kiwi.pm --- old/obs-build-20180329/Build/Kiwi.pm 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/Build/Kiwi.pm 2018-06-29 07:58:19.000000000 +0200 @@ -32,17 +32,18 @@ return grep(delete($h{$_}), @_); } -sub findFallBackArchs { - my ($fallbackArchXML, $arch) = @_; - my @fa; - - for my $a (@{$fallbackArchXML->{'arch'}||[]}) { - if ( $a->{'id'} eq $arch && $a->{'fallback'} ) { - @fa = unify( $a->{'fallback'}, findFallBackArchs($fallbackArchXML, $a->{'fallback'})); - } +sub expandFallBackArchs { + my ($fallbackArchXML, @archs) = @_; + my %fallbacks; + for (@{$fallbackArchXML->{'arch'} || []}) { + $fallbacks{$_->{'id'}} = $_->{'fallback'} if $_->{id} && $_->{'fallback'}; + } + my @out; + while (@archs) { + push @out, shift @archs; + push @archs, delete $fallbacks{$out[-1]} if $fallbacks{$out[-1]}; } - - return @fa + return unify(@out); } # sles10 perl does not have the version.pm @@ -50,15 +51,166 @@ sub versionstring { my ($str) = @_; my $result = 0; - $result = $result * 100 + $_ for split (/\./, $str); + $result = $result * 100 + $_ for split (/\./, $str, 2); return $result; } +my $schemaversion56 = versionstring('5.6'); + +sub kiwiparse_product { + my ($kiwi, $xml, $arch, $buildflavor) = @_; + + my $ret = {}; + my @repos; + my %repoprio; # XXX: unused + my @packages; + my @requiredarch; + my @badarch; + my $obsexclusivearch; + my $obsexcludearch; + $obsexclusivearch = $1 if $xml =~ /^\s*<!--\s+OBS-ExclusiveArch:\s+(.*)\s+-->\s*$/im; + $obsexcludearch = $1 if $xml =~ /^\s*<!--\s+OBS-ExcludeArch:\s+(.*)\s+-->\s*$/im; + + $ret->{'name'} = $kiwi->{'name'} if $kiwi->{'name'}; + $ret->{'filename'} = $kiwi->{'name'} if $kiwi->{'name'}; + my $description = (($kiwi->{'description'} || [])->[0]) || {}; + if (!$ret->{'name'} && $description->{'specification'}) { + $ret->{'name'} = $description->{'specification'}->[0]->{'_content'}; + } + + # parse the preferences section + my $preferences = $kiwi->{'preferences'} || []; + die("products must have exactly one preferences element\n") unless @$preferences == 1; + # take default version setting + if ($preferences->[0]->{'version'}) { + $ret->{'version'} = $preferences->[0]->{'version'}->[0]->{'_content'}; + } + die("products must have exactly one type element in the preferences\n") unless @{$preferences->[0]->{'type'} || []} == 1; + my $preftype = $preferences->[0]->{'type'}->[0]; + if (defined $preftype->{'image'}) { + # for kiwi 4.1 and 5.x + die("products must use type 'product'\n") unless $preftype->{'image'} eq 'product'; + } else { + # for kiwi 3.8 and before + die("products must use type 'product'\n") unless $preftype->{'_content'} eq 'product'; + } + push @packages, "kiwi-filesystem:$preftype->{'filesystem'}" if $preftype->{'filesystem'}; + die("boot type not supported in products\n") if defined $preftype->{'boot'}; + + my $instsource = ($kiwi->{'instsource'} || [])->[0]; + die("products must have an instsource element\n") unless $instsource; + + # get repositories + for my $repository (sort {$a->{priority} <=> $b->{priority}} @{$instsource->{'instrepo'} || []}) { + my $kiwisource = ($repository->{'source'} || [])->[0]; + if ($kiwisource->{'path'} eq 'obsrepositories:/') { + # special case, OBS will expand it. + push @repos, '_obsrepositories'; + next; + } + if ($kiwisource->{'path'} =~ /^obs:\/\/\/?([^\/]+)\/([^\/]+)\/?$/) { + push @repos, "$1/$2"; + } else { + my $prp; + $prp = $urlmapper->($kiwisource->{'path'}) if $urlmapper; + die("instsource repo url not using obs:/ scheme: $kiwisource->{'path'}\n") unless $prp; + push @repos, $prp; + } + } + + $ret->{'sourcemedium'} = -1; + $ret->{'debugmedium'} = -1; + if ($instsource->{'productoptions'}) { + my $productoptions = $instsource->{'productoptions'}->[0] || {}; + for my $po (@{$productoptions->{'productvar'} || []}) { + $ret->{'drop_repository'} = $po->{'_content'} if $po->{'name'} eq 'DROP_REPOSITORY'; + $ret->{'version'} = $po->{'_content'} if $po->{'name'} eq 'VERSION'; + } + for my $po (@{$productoptions->{'productoption'} || []}) { + $ret->{'sourcemedium'} = $po->{'_content'} if $po->{'name'} eq 'SOURCEMEDIUM'; + $ret->{'debugmedium'} = $po->{'_content'} if $po->{'name'} eq 'DEBUGMEDIUM'; + } + } + if ($instsource->{'architectures'}) { + my $architectures = $instsource->{'architectures'}->[0] || {}; + for my $ra (@{$architectures->{'requiredarch'} || []}) { + push @requiredarch, $ra->{'ref'} if defined $ra->{'ref'}; + } + } + + # Find packages and possible additional required architectures + my @additionalarchs; + my @pkgs; + push @pkgs, @{$instsource->{'metadata'}->[0]->{'repopackage'} || []} if $instsource->{'metadata'}; + push @pkgs, @{$instsource->{'repopackages'}->[0]->{'repopackage'} || []} if $instsource->{'repopackages'}; + @pkgs = unify(@pkgs); + for my $package (@pkgs) { + # filter packages, which are not targeted for the wanted plattform + if ($package->{'arch'}) { + my $valid; + for my $ma (@requiredarch) { + for my $pa (split(',', $package->{'arch'})) { + $valid = 1 if $ma eq $pa; + } + } + next unless $valid; + } + + # not nice, but optimizes our build dependencies + # FIXME: design a real blacklist option in kiwi + if ($package->{'onlyarch'} && $package->{'onlyarch'} eq 'skipit') { + push @packages, "-$package->{'name'}"; + next; + } + push @packages, "-$package->{'replaces'}" if $package->{'replaces'}; + + # we need this package + push @packages, $package->{'name'}; + + # find the maximal superset of possible required architectures + push @additionalarchs, split(',', $package->{'addarch'}) if $package->{'addarch'}; + push @additionalarchs, split(',', $package->{'onlyarch'}) if $package->{'onlyarch'}; + } + @requiredarch = unify(@requiredarch, @additionalarchs); + + #### FIXME: kiwi files have no informations where to get -32bit packages from + push @requiredarch, "i586" if grep {/^ia64/} @requiredarch; + push @requiredarch, "i586" if grep {/^x86_64/} @requiredarch; + push @requiredarch, "ppc" if grep {/^ppc64/} @requiredarch; + push @requiredarch, "s390" if grep {/^s390x/} @requiredarch; + + @requiredarch = expandFallBackArchs($instsource->{'architectures'}->[0], @requiredarch); + + push @packages, "kiwi-packagemanager:instsource"; + + push @requiredarch, split(' ', $obsexclusivearch) if $obsexclusivearch; + push @badarch , split(' ', $obsexcludearch) if $obsexcludearch; + + $ret->{'exclarch'} = [ unify(@requiredarch) ] if @requiredarch; + $ret->{'badarch'} = [ unify(@badarch) ] if @badarch; + $ret->{'deps'} = [ unify(@packages) ]; + $ret->{'path'} = [ unify(@repos) ]; + $ret->{'imagetype'} = [ 'product' ]; + for (@{$ret->{'path'} || []}) { + my @s = split('/', $_, 2); + $_ = {'project' => $s[0], 'repository' => $s[1]}; + $_->{'priority'} = $repoprio{"$s[0]/$s[1]"} if $repoextras && defined $repoprio{"$s[0]/$s[1]"}; + } + return $ret; +} + sub kiwiparse { - my ($xml, $arch, $count, $buildflavor) = @_; + my ($xml, $arch, $buildflavor, $count) = @_; $count ||= 0; die("kiwi config inclusion depth limit reached\n") if $count++ > 10; + my $kiwi = Build::SimpleXML::parse($xml); + die("not a kiwi config\n") unless $kiwi && $kiwi->{'image'}; + $kiwi = $kiwi->{'image'}->[0]; + + # check if this is a product, we currently test for the 'instsource' element + return kiwiparse_product($kiwi, $xml, $arch, $buildflavor) if $kiwi->{'instsource'}; + my $ret = {}; my @types; my @repos; @@ -67,10 +219,6 @@ my @containerrepos; my @packages; my @extrasources; - my @requiredarch; - my @badarch; - my $schemaversion = 0; - my $schemaversion56 = versionstring('5.6'); my $obsexclusivearch; my $obsexcludearch; my $obsprofiles; @@ -80,10 +228,7 @@ if ($obsprofiles) { $obsprofiles = [ grep {defined($_)} map {$_ eq '@BUILD_FLAVOR@' ? $buildflavor : $_} split(' ', $obsprofiles) ]; } - my $kiwi = Build::SimpleXML::parse($xml); - die("not a kiwi config\n") unless $kiwi && $kiwi->{'image'}; - $kiwi = $kiwi->{'image'}->[0]; - $schemaversion = versionstring($kiwi->{'schemaversion'}) if $kiwi->{'schemaversion'}; + my $schemaversion = $kiwi->{'schemaversion'} ? versionstring($kiwi->{'schemaversion'}) : 0; $ret->{'name'} = $kiwi->{'name'} if $kiwi->{'name'}; $ret->{'filename'} = $kiwi->{'name'} if $kiwi->{'name'}; my $description = (($kiwi->{'description'} || [])->[0]) || {}; @@ -91,7 +236,9 @@ $ret->{'name'} = $description->{'specification'}->[0]->{'_content'}; } - # do obsprofiles arch filtering + # usedprofiles also include direct wanted profile targets and indirect required profiles + my %usedprofiles; + # obsprofiles arch filtering if ($obsprofiles && $arch && $kiwi->{'profiles'} && $kiwi->{'profiles'}->[0]->{'profile'}) { my %obsprofiles = map {$_ => 1} @$obsprofiles; for my $prof (@{$kiwi->{'profiles'}[0]->{'profile'}}) { @@ -114,6 +261,13 @@ } } $obsprofiles = [ grep {$obsprofiles{$_}} @$obsprofiles ]; + for my $prof (@{$kiwi->{'profiles'}[0]->{'profile'}}) { + next unless $obsprofiles{$prof->{'name'}}; + $usedprofiles{$prof->{'name'}} = 1; + for my $req (@{$prof->{'requires'}}) { + $usedprofiles{$req->{'profile'}} = 1; + } + } } # take default version setting @@ -124,8 +278,7 @@ my $containerconfig; for my $pref (@{$preferences || []}) { if ($obsprofiles && $pref->{'profiles'}) { - my %obsprofiles = map {$_ => 1} @$obsprofiles; - next unless grep {$obsprofiles{$_}} split(",", $pref->{'profiles'}); + next unless grep {$usedprofiles{$_}} split(",", $pref->{'profiles'}); } for my $type (@{$pref->{'type'} || []}) { next unless @{$pref->{'type'}} == 1 || !$type->{'optional'}; @@ -144,25 +297,28 @@ if ($type->{'derived_from'}) { my $derived = $type->{'derived_from'}; my ($name, $prp); - if ($derived =~ /^obs:\/{1,3}([^\/]+)\/([^\/]+)\/(.*)(?:#([^\#\/]+))$/) { - $name = defined($4) ? "$3:$4" : "$3"; + if ($derived =~ /^obs:\/{1,3}([^\/]+)\/([^\/]+)\/(.*?)(?:#([^\#\/]+))?$/) { + $name = defined($4) ? "$3:$4" : "$3:latest"; $prp = "$1/$2"; - } elsif ($derived =~ /^obsrepositories:\/{1,3}([^\/].*)(?:#([^\#\/]+))$/) { - $name = defined($2) ? "$1:$2" : "$1"; - } elsif ($derived !~ /^file:/ && $derived =~ /^(.*)\/([^\/]+)(?:#([^\#\/]+))$/) { + } elsif ($derived =~ /^obsrepositories:\/{1,3}([^\/].*?)(?:#([^\#\/]+))?$/) { + $name = defined($2) ? "$1:$2" : "$1:latest"; + } elsif ($derived =~ /^file:/) { + next; # just ignore and hope + } elsif ($derived =~ /^(.*)\/([^\/]+?)(?:#([^\#\/]+))?$/) { my $url = $1; - $name = defined($3) ? "$2:$3" : "$2"; + $name = defined($3) ? "$2:$3" : "$2:latest"; $prp = $urlmapper->($url) if $urlmapper; # try again with one element moved from url to name - if (!$prp && $derived =~ /^(.*)\/([^\/]+\/[^\/]+)(?:#([^\#\/]+))$/) { + if (!$prp && $derived =~ /^(.*)\/([^\/]+\/[^\/]+?)(?:#([^\#\/]+))?$/) { $url = $1; - $name = defined($3) ? "$2:$3" : "$2"; + $name = defined($3) ? "$2:$3" : "$2:latest"; $prp = $urlmapper->($url) if $urlmapper; } - die("derived_from url not using obs:/ scheme: $derived\n") unless $prp; + undef $name unless $prp; } - push @packages, "container:$name" if defined $name; - push @containerrepos, $prp if $prp; + die("derived_from url not using obs:/ scheme: $derived\n") unless defined $name; + push @packages, "container:$name"; + push @containerrepos, $prp if $prp; } push @packages, "kiwi-filesystem:$type->{'filesystem'}" if $type->{'filesystem'}; @@ -172,7 +328,7 @@ my ($bootxml, $xsrc) = $bootcallback->($1, $2); next unless $bootxml; push @extrasources, $xsrc if $xsrc; - my $bret = kiwiparse($bootxml, $arch, $count, $buildflavor); + my $bret = kiwiparse($bootxml, $arch, $buildflavor, $count); push @bootrepos, map {"$_->{'project'}/$_->{'repository'}"} @{$bret->{'path'} || []}; push @packages, @{$bret->{'deps'} || []}; push @extrasources, @{$bret->{'extrasource'} || []}; @@ -184,44 +340,7 @@ } } - my $instsource = ($kiwi->{'instsource'} || [])->[0]; - if ($instsource) { - for my $repository(sort {$a->{priority} <=> $b->{priority}} @{$instsource->{'instrepo'} || []}) { - my $kiwisource = ($repository->{'source'} || [])->[0]; - if ($kiwisource->{'path'} eq 'obsrepositories:/') { - # special case, OBS will expand it. - push @repos, '_obsrepositories'; - next; - } - if ($kiwisource->{'path'} =~ /^obs:\/\/\/?([^\/]+)\/([^\/]+)\/?$/) { - push @repos, "$1/$2"; - } else { - my $prp; - $prp = $urlmapper->($kiwisource->{'path'}) if $urlmapper; - die("instsource repo url not using obs:/ scheme: $kiwisource->{'path'}\n") unless $prp; - push @repos, $prp; - } - } - $ret->{'sourcemedium'} = -1; - $ret->{'debugmedium'} = -1; - if ($instsource->{'productoptions'}) { - my $productoptions = $instsource->{'productoptions'}->[0] || {}; - for my $po (@{$productoptions->{'productvar'} || []}) { - $ret->{'drop_repository'} = $po->{'_content'} if $po->{'name'} eq 'DROP_REPOSITORY'; - $ret->{'version'} = $po->{'_content'} if $po->{'name'} eq 'VERSION'; - } - for my $po (@{$productoptions->{'productoption'} || []}) { - $ret->{'sourcemedium'} = $po->{'_content'} if $po->{'name'} eq 'SOURCEMEDIUM'; - $ret->{'debugmedium'} = $po->{'_content'} if $po->{'name'} eq 'DEBUGMEDIUM'; - } - } - if ($instsource->{'architectures'}) { - my $a = $instsource->{'architectures'}->[0] || {}; - for my $ra (@{$a->{'requiredarch'} || []}) { - push @requiredarch, $ra->{'ref'} if defined $ra->{'ref'}; - } - } - } + die("image contains 'product' type\n") if grep {$_ eq 'product'} @types; my $packman = $preferences->[0]->{'packagemanager'}->[0]->{'_content'} || ''; @@ -262,8 +381,7 @@ $repoprio{$prp} = $repository->{'sortprio'} if defined $repository->{'priority'}; } - # Find packages and possible additional required architectures - my @additionalarchs; + # Find packages for the image my @pkgs; my $patterntype; for my $packages (@{$kiwi->{'packages'}}) { @@ -271,10 +389,9 @@ # we could skip the sections also when no profile is used, # but don't to stay backward compatible if ($obsprofiles && $packages->{'profiles'}) { - my %obsprofiles = map {$_ => 1} @$obsprofiles; my @section_profiles = split(",", $packages->{'profiles'}); - next unless grep {$obsprofiles{$_}} @section_profiles; + next unless grep {$usedprofiles{$_}} @section_profiles; } $patterntype ||= $packages->{'patternType'}; @@ -293,78 +410,30 @@ } } $patterntype ||= 'onlyRequired'; - if ($instsource) { - push @pkgs, @{$instsource->{'metadata'}->[0]->{'repopackage'} || []} if $instsource->{'metadata'}; - push @pkgs, @{$instsource->{'repopackages'}->[0]->{'repopackage'} || []} if $instsource->{'repopackages'}; - } @pkgs = unify(@pkgs); for my $package (@pkgs) { - # filter packages, which are not targeted for the wanted plattform + # filter packages which are not targeted for the wanted plattform if ($package->{'arch'}) { my $valid; - if (@requiredarch) { - # this is a product - for my $ma (@requiredarch) { - for my $pa (split(",", $package->{'arch'})) { - $valid = 1 if $ma eq $pa; - } - } - } else { - # live appliance - my $ma = $arch; - $ma =~ s/i[456]86/i386/; - for my $pa (split(",", $package->{'arch'})) { - $pa =~ s/i[456]86/i386/; - $valid = 1 if $ma eq $pa; - } + my $ma = $arch; + $ma =~ s/i[456]86/i386/; + for my $pa (split(",", $package->{'arch'})) { + $pa =~ s/i[456]86/i386/; + $valid = 1 if $ma eq $pa; } next unless $valid; } - - # not nice, but optimizes our build dependencies - # FIXME: design a real blacklist option in kiwi - if ($package->{'onlyarch'} && $package->{'onlyarch'} eq "skipit") { - push @packages, "-".$package->{'name'}; - next; - } # handle replaces as buildignore - if ($package->{'replaces'}) { - push @packages, "-".$package->{'replaces'}; - } + push @packages, "-$package->{'replaces'}" if $package->{'replaces'}; # we need this package push @packages, $package->{'name'}; - - # find the maximal superset of possible required architectures - push @additionalarchs, split(',', $package->{'addarch'}) if $package->{'addarch'}; - push @additionalarchs, split(',', $package->{'onlyarch'}) if $package->{'onlyarch'}; } - @requiredarch = unify(@requiredarch, @additionalarchs); + push @packages, "kiwi-packagemanager:$packman"; + push @packages, "--dorecommends--", "--dosupplements--" if $patterntype && $patterntype eq 'plusRecommended'; - #### FIXME: kiwi files have no informations where to get -32bit packages from - push @requiredarch, "i586" if grep {/^ia64/} @requiredarch; - push @requiredarch, "i586" if grep {/^x86_64/} @requiredarch; - push @requiredarch, "ppc" if grep {/^ppc64/} @requiredarch; - push @requiredarch, "s390" if grep {/^s390x/} @requiredarch; - - my @fallbackarchs; - for my $arch (@requiredarch) { - push @fallbackarchs, findFallBackArchs($instsource->{'architectures'}[0], $arch) if $instsource->{'architectures'}[0]; - } - @requiredarch = unify(@requiredarch, @fallbackarchs); - - if (!$instsource) { - push @packages, "kiwi-packagemanager:$packman"; - push @packages, "--dorecommends--", "--dosupplements--" if $patterntype && $patterntype eq 'plusRecommended'; - } else { - push @packages, "kiwi-packagemanager:instsource"; - } - - push @requiredarch, split(' ', $obsexclusivearch) if $obsexclusivearch; - push @badarch , split(' ', $obsexcludearch) if $obsexcludearch; - - $ret->{'exclarch'} = [ unify(@requiredarch) ] if @requiredarch; - $ret->{'badarch'} = [ unify(@badarch) ] if @badarch; + $ret->{'exclarch'} = [ unify(split(' ', $obsexclusivearch)) ] if $obsexclusivearch; + $ret->{'badarch'} = [ unify(split(' ', $obsexcludearch)) ] if $obsexcludearch; $ret->{'deps'} = [ unify(@packages) ]; $ret->{'path'} = [ unify(@repos, @bootrepos) ]; $ret->{'containerpath'} = [ unify(@containerrepos) ] if @containerrepos; @@ -380,7 +449,7 @@ $_ = {'project' => $s[0], 'repository' => $s[1]}; } $ret->{'imagerepos'} = \@imagerepos if @imagerepos; - if (!$instsource && $containerconfig) { + if ($containerconfig) { my $containername = $containerconfig->{'name'}; my $containertags = $containerconfig->{'tag'}; $containertags = [ $containertags ] if defined($containertags) && !ref($containertags); @@ -413,12 +482,10 @@ close F; $cf ||= {}; my $d; - eval { - $d = kiwiparse($xml, ($cf->{'arch'} || ''), 0, $cf->{'buildflavor'}); - }; + eval { $d = kiwiparse($xml, ($cf->{'arch'} || ''), $cf->{'buildflavor'}, 0) }; if ($@) { my $err = $@; - $err =~ s/^\n$//s; + chomp $err; return {'error' => $err}; } return $d; @@ -441,11 +508,16 @@ } sub showcontainerinfo { - my $disturl; + my ($disturl, $arch, $buildflavor); (undef, $disturl) = splice(@ARGV, 0, 2) if @ARGV > 2 && $ARGV[0] eq '--disturl'; + (undef, $arch) = splice(@ARGV, 0, 2) if @ARGV > 2 && $ARGV[0] eq '--arch'; + (undef, $buildflavor) = splice(@ARGV, 0, 2) if @ARGV > 2 && $ARGV[0] eq '--buildflavor'; my ($fn, $image) = @ARGV; local $urlmapper = sub { return $_[0] }; - my $d = parse({}, $fn); + my $cf = {}; + $cf->{'arch'} = $arch if defined $arch; + $cf->{'buildflavor'} = $buildflavor if defined $buildflavor; + my $d = parse($cf, $fn); die("$d->{'error'}\n") if $d->{'error'}; $image =~ s/.*\/// if defined $image; my $release; @@ -459,6 +531,7 @@ push @repos, "{ \"url\": \"$repo->{'url'}\" }"; } } + my $buildtime = time(); print "{\n"; print " \"name\": \"$d->{'name'}\""; print ",\n \"version\": \"$d->{'version'}\"" if defined $d->{'version'}; @@ -467,6 +540,7 @@ print ",\n \"repos\": [ ".join(', ', @repos)." ]" if @repos; print ",\n \"file\": \"$image\"" if defined $image; print ",\n \"disturl\": \"$disturl\"" if defined $disturl; + print ",\n \"buildtime\": $buildtime"; print "\n}\n"; } @@ -476,7 +550,6 @@ return {}; } - sub queryhdrmd5 { my ($bin) = @_; die("Build::Kiwi::queryhdrmd5 unimplemented.\n"); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/Build/Rpm.pm new/obs-build-20180629/Build/Rpm.pm --- old/obs-build-20180329/Build/Rpm.pm 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/Build/Rpm.pm 2018-06-29 07:58:19.000000000 +0200 @@ -1189,16 +1189,14 @@ my ($d) = @_; my $n = ''; my $bl = 0; - while ($d =~ /^([^ ,\)]+)/) { + while ($d =~ /^([^ ,\(\)]*)/) { $n .= $1; - $d = substr($d, length($n)); - if ($d =~ /^\(/) { - $n .= '('; - $bl++; - $d = substr($d, 1); - } elsif ($d =~ /\)/) { - last if $bl-- <= 0; - } + $d = substr($d, length($1)); + last unless $d =~ /^([\(\)])/; + $bl += $1 eq '(' ? 1 : -1; + last if $bl < 0; + $n .= $1; + $d = substr($d, 1); } return $n; } @@ -1272,7 +1270,7 @@ my $r1 = testcaseformat_rec($r->[1], 1); if (($op == 3 || $op == 4) && @$r == 4) { $r1 = "$r1 $top " . testcaseformat_rec($r->[2], 1); - $top = 'ELSE'; + $top = '<ELSE>'; } my $addparens2 = 1; $addparens2 = 0 if $r->[2]->[0] == $op && ($op == 1 || $op == 2 || $op == 6); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/Build.pm new/obs-build-20180629/Build.pm --- old/obs-build-20180329/Build.pm 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/Build.pm 2018-06-29 07:58:19.000000000 +0200 @@ -612,12 +612,10 @@ @sysdeps = @{$config->{'substitute'}->{'system-packages:kiwi-image'} || []}; @sysdeps = @{$config->{'substitute'}->{'kiwi-setup:image'} || []} unless @sysdeps; @sysdeps = @{$subst_defaults{'system-packages:kiwi-image'} || []} unless @sysdeps; - push @sysdeps, @$extradeps if $extradeps; } elsif ($buildtype eq 'kiwi-product') { @sysdeps = @{$config->{'substitute'}->{'system-packages:kiwi-product'} || []}; @sysdeps = @{$config->{'substitute'}->{'kiwi-setup:product'} || []} unless @sysdeps; @sysdeps = @{$subst_defaults{'system-packages:kiwi-product'} || []} unless @sysdeps; - push @sysdeps, @$extradeps if $extradeps; } elsif ($buildtype eq 'docker') { @sysdeps = @{$config->{'substitute'}->{'system-packages:docker'} || []} unless @sysdeps; @sysdeps = @{$subst_defaults{'system-packages:docker'} || []} unless @sysdeps; @@ -628,7 +626,8 @@ @sysdeps = @{$config->{'substitute'}->{'system-packages:deltarpm'} || []}; @sysdeps = @{$subst_defaults{'system-packages:deltarpm'} || []} unless @sysdeps; } - return () unless @sysdeps; + return () unless @sysdeps; # no extra build environment used + push @sysdeps, @$extradeps if $extradeps; if ($config->{'expandflags:preinstallexpand'} && !$config->{'preinstallisexpanded'}) { my $err = expandpreinstalls($config); return (undef, $err) if $err; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/build new/obs-build-20180629/build --- old/obs-build-20180329/build 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/build 2018-06-29 07:58:19.000000000 +0200 @@ -172,8 +172,6 @@ --no-checks Do not run checks (postbuild and %check) - --lint Run rpmlint after build. - --logfile logfile Capture build output to logfile. Defaults to .build.log in the build root for non-VM builds. @@ -332,7 +330,8 @@ --vm-disk-size SIZEINMB --vm-swap-size SIZEINMB --vm-disk-filesystem TYPE - Defaults for automatic setup of VM root/swap files + Defaults for automatic setup of VM root/swap files. + May get overruled by build config vmfstype build flag. --vm-memory SIZEINMB Set amount of RAM for VMs @@ -851,6 +850,7 @@ shift ;; -lint) + # OBSOLETE: just keeping it for compability DO_LINT=true ;; -baselibs) @@ -1308,6 +1308,10 @@ imagetype=$(perl -I$BUILD_DIR -MBuild::Kiwi -e Build::Kiwi::show $RECIPEFILE imagetype) test "$imagetype" = product || BUILD_USER=root fi + # same with docker and fissile builds + if test "$BUILDTYPE" = docker -o "$BUILDTYPE" = fissile ; then + BUILD_USER=root + fi # fixup passwd/group if test $BUILD_USER = abuild ; then diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/build-recipe-docker new/obs-build-20180629/build-recipe-docker --- old/obs-build-20180329/build-recipe-docker 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/build-recipe-docker 2018-06-29 07:58:19.000000000 +0200 @@ -118,9 +118,16 @@ done # patch in obs-docker-support helper - sed -i '/^[ ]*[fF][rR][oO][mM]/a COPY .obs-docker-support /usr/local/sbin/obs-docker-support\nRUN obs-docker-support --install' $BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE + sed -i '/^[ ]*[fF][rR][oO][mM]/a COPY .obs-docker-support /usr/local/sbin/obs-docker-support\nRUN obs-docker-support --install' "$BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE" echo >> $BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE - echo 'RUN obs-docker-support --uninstall' >> $BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE + if test -n "$(sed -ne '/^[ ]*[uU][sS][eE][rR][ ]/p' "$BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE")" ; then + sed -i '1,/^[ ]*[uU][sS][eE][rR][ ]/{ +/^[ ]*[uU][sS][eE][rR][ ]/i RUN obs-docker-support --uninstall +}' "$BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE" + else + echo 'RUN obs-docker-support --uninstall' >> "$BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE" + fi + test -n "$DISTURL" && echo "LABEL org.openbuildservice.disturl=$DISTURL" >> "$BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE" # now do the build echo "Building image $ALLTAGS" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/build-recipe-kiwi new/obs-build-20180629/build-recipe-kiwi --- old/obs-build-20180329/build-recipe-kiwi 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/build-recipe-kiwi 2018-06-29 07:58:19.000000000 +0200 @@ -523,6 +523,14 @@ --set-repopriority=*|--set-repoprio=*) set_repo_priority="${i#*=}" ;; --main:*) KIWI_MAIN_PARAMETERS="$KIWI_MAIN_PARAMETERS --${i#--main:}" ;; --bundle:*) KIWI_BUNDLE_PARAMETERS="$KIWI_BUNDLE_PARAMETERS --${i#--bundle:}" ;; + --set-container-derived-from=dir://./containers/*.tar) + # old kiwi 9 versions cannot handle uncompressed containers + if test -f "$BUILD_ROOT/$TOPDIR/SOURCES/containers/${i#*containers/}" ; then + gzip -f -1 < "$BUILD_ROOT/$TOPDIR/SOURCES/containers/${i#*containers/}" > "$BUILD_ROOT/$TOPDIR/SOURCES/containers/${i#*containers/}.gz" || cleanup_and_exit 1 "container compression" + i="$i.gz" + fi + KIWI_BUILD_PARAMETERS="$KIWI_BUILD_PARAMETERS $i" + ;; # everything else just gues through to kiwi build *) KIWI_BUILD_PARAMETERS="$KIWI_BUILD_PARAMETERS $i" ;; esac @@ -597,6 +605,29 @@ fi } +postprocess_kiwi_containers() { + local r + + # uncompress containers, the compression is useless and just slows down further processing + for r in $BUILD_ROOT/$TOPDIR/KIWI/*.tar.xz ; do + test -e "$r" || continue + r="${r%.xz}" + xzdec <"$r.xz" >"$r" || cleanup_and_exit 1 "xzdec failed" + if test -e "$r.xz.sha256" ; then + # also generate sha256sum + (cd "$BUILD_ROOT/$TOPDIR/KIWI" && sha256sum "${r##*/}") > "$r.sha256" + fi + rm -f "$r.xz" "$r.xz.sha256" + done + + # create container info + local disturlarg=() + test -n "$DISTURL" && disturlarg=("--disturl" "$DISTURL") + for r in $BUILD_ROOT/$TOPDIR/KIWI/*.tar ; do + test -e "$r" && perl -I$BUILD_DIR -MBuild::Kiwi -e Build::Kiwi::showcontainerinfo -- "${disturlarg[@]}" --arch "${BUILD_ARCH%%:*}" --buildflavor "$BUILD_FLAVOR" $BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE "$r" > "${r%.tar}.containerinfo" + done +} + recipe_build_kiwi() { imagetype=$(kiwi_query_recipe imagetype) imagename=$(kiwi_query_recipe filename) @@ -681,20 +712,17 @@ # Hook for running post kiwi build scripts like QA scripts if installed if test -x $BUILD_ROOT/usr/lib/build/kiwi_post_run ; then - if ! chroot $BUILD_ROOT su -c /usr/lib/build/kiwi_post_run; then - echo "/usr/lib/build/kiwi_post_run script failed!" - cleanup_and_exit 1 - fi + if ! chroot $BUILD_ROOT su -c /usr/lib/build/kiwi_post_run ; then + cleanup_and_exit 1 "/usr/lib/build/kiwi_post_run script failed!" + fi fi - if test "$imagetype" = docker ; then - # get container info - disturlarg=() - test -n "$DISTURL" && disturlarg=("--disturl" "$DISTURL") - for r in $BUILD_ROOT/$TOPDIR/KIWI/*.tar.xz ; do - test -e "$r" && perl -I$BUILD_DIR -MBuild::Kiwi -e Build::Kiwi::showcontainerinfo -- "${disturlarg[@]}" $BUILD_ROOT/$TOPDIR/SOURCES/$RECIPEFILE "$r" > "${r%.tar.xz}.containerinfo" - done - fi + for imgtype in $imagetype ; do + if test "$imgtype" = docker ; then + postprocess_kiwi_containers + break + fi + done } recipe_resultdirs_kiwi() { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/build-vm new/obs-build-20180629/build-vm --- old/obs-build-20180329/build-vm 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/build-vm 2018-06-29 07:58:19.000000000 +0200 @@ -355,7 +355,7 @@ vm_wipe_$VM_TYPE "$@" if test -n "$VM_ROOT" -a "$VM_ROOT_TYPE" = file ; then - rm -f "$VM_ROOT" "${VM_ROOT}.monitor" + rm -f "$VM_ROOT" fi if test -n "$VM_SWAP" -a "$VM_SWAP_TYPE" = file ; then rm -f "$VM_SWAP" @@ -429,7 +429,7 @@ WATCHDOG_TIMEOUT=300 BUILD_OPTIONS_PARSED= while sleep 5 ; do - WATCH=`grep -a "### VM INTERACTION" "$LOGFILE" | tail -n 1` + WATCH=$(grep -a "### VM INTERACTION" "$LOGFILE" | tr '\0' a | tail -n 1) case $WATCH in *VM\ INTERACTION\ START*) test -n "$WATCHDOG_START" || WATCHDOG_START=`date +%s` ;; *VM\ INTERACTION\ END*) WATCHDOG_START= ;; @@ -603,9 +603,13 @@ } vm_set_filesystem_type() { - if test -z "$VMDISK_FILESYSTEM" -a -n "$BUILD_DIST" ; then - VMDISK_FILESYSTEM=`queryconfig --dist "$BUILD_DIST" --configdir "$CONFIG_DIR" --archpath "$BUILD_ARCH" buildflags vmfstype` - fi + vmfstype="" + if test -n "$BUILD_DIST" ; then + # testing for build specific filesystem, which are more important then worker defaults + vmfstype=`queryconfig --dist "$BUILD_DIST" --configdir "$CONFIG_DIR" --archpath "$BUILD_ARCH" buildflags vmfstype` + fi + test -n "$vmfstype" && VMDISK_FILESYSTEM="$vmfstype" + # use either commandline specified fs or ext3 as fallback test -n "$VMDISK_FILESYSTEM" || VMDISK_FILESYSTEM=ext3 } @@ -917,7 +921,7 @@ vm_attach_root if test -n "$VM_SWAP" ; then vm_attach_swap - BUILDSTATUS=`dd if="$VM_SWAP" bs=12 count=1 2>/dev/null` + BUILDSTATUS=$(dd if="$VM_SWAP" bs=12 count=1 2>/dev/null | tr '\0' a) case $BUILDSTATUS in BUILDSTATUS[02]) mkdir -p $BUILD_ROOT/.build.packages @@ -993,7 +997,7 @@ echo "... saving built packages" swapoff "$VM_SWAP" pushd "$BUILD_ROOT$TOPDIR" >/dev/null - find "$@" -print0 | computeblocklists --padstart 512 --padend 512 -v --manifest - -0 > "$VM_SWAP" + find "$@" -print0 | computeblocklists --padstart 512 --padend 512 -v --manifest - -0 > "$VM_SWAP" || cleanup_and_exit 1 popd >/dev/null fi } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/build-vm-docker new/obs-build-20180629/build-vm-docker --- old/obs-build-20180329/build-vm-docker 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/build-vm-docker 2018-06-29 07:58:19.000000000 +0200 @@ -32,7 +32,7 @@ docker rm "$name" >/dev/null 2>&1 || true docker run \ --rm --name "$name" --cap-add=sys_admin --cap-add=MKNOD --net=none \ - -v "$BUILD_ROOT:/mnt" busybox /bin/chroot /mnt "$vm_init_script" + --mount "type=bind,source=$BUILD_ROOT,destination=/mnt" busybox /bin/chroot /mnt "$vm_init_script" BUILDSTATUS="$?" test "$BUILDSTATUS" != 255 || BUILDSTATUS=3 cleanup_and_exit "$BUILDSTATUS" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/build-vm-kvm new/obs-build-20180629/build-vm-kvm --- old/obs-build-20180329/build-vm-kvm 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/build-vm-kvm 2018-06-29 07:58:19.000000000 +0200 @@ -243,8 +243,10 @@ elif test -n "$VM_CONSOLE_INPUT" ; then qemu_args=("${qemu_args[@]}" -serial mon:stdio) else - mkfifo "${VM_ROOT}.monitor" - qemu_args=("${qemu_args[@]}" -serial stdio -chardev socket,id=monitor,server,nowait,path="${VM_ROOT}.monitor" -mon chardev=monitor,mode=readline) + mkdir -p "${VM_ROOT}.qemu" + mkfifo "${VM_ROOT}.qemu/monitor" + chown "$VM_USER" "${VM_ROOT}.qemu" + qemu_args=("${qemu_args[@]}" -serial stdio -chardev socket,id=monitor,server,nowait,path="${VM_ROOT}.qemu/monitor" -mon chardev=monitor,mode=readline) fi if test -n "$BUILD_JOBS" -a "$icecream" = 0 -a -z "$BUILD_THREADS" ; then @@ -287,6 +289,8 @@ export QEMU_AUDIO_DRV=none # we do not want to have sound inside the VMs echo "$@" "$@" + qemu_ret=$? + test "$qemu_ret" = "137" && cleanup_and_exit 3 "qemu got SIGKILL" } vm_kill_kvm() { @@ -330,7 +334,7 @@ vm_sysrq_kvm() { perl -e 'use Socket; socket(SOCK, PF_UNIX, SOCK_STREAM, 0) || die("socket: $!\n"); - connect(SOCK, sockaddr_un("'"$VM_ROOT.monitor"'")) || die("connect: $!\n"); + connect(SOCK, sockaddr_un("'"$VM_ROOT.qemu/monitor"'")) || die("connect: $!\n"); print SOCK "sendkey alt-print-'$1'\n";' } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/computeblocklists new/obs-build-20180629/computeblocklists --- old/obs-build-20180329/computeblocklists 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/computeblocklists 2018-06-29 07:58:19.000000000 +0200 @@ -13,7 +13,7 @@ ################################################################ # -# Copyright (c) 1995-2014 SUSE Linux Products GmbH +# Copyright (c) 1995-2018 SUSE Linux Products GmbH # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or 3 as @@ -33,7 +33,163 @@ use strict; -my ($opt_padstart, $opt_padend, $opt_verbose, $opt_manifest, $opt_mani0); +# the ioctls we use +my $FIBMAP = 1; +my $FIGETBSZ = 2; +my $FIEMAP = 0xc020660b; # _IOWR('f', 11, struct fiemap) + +# powerpc, mips, sparc, and alpha have 3 direction bits +# and represent _IOC_NONE as 1 instead of 0. +sub alt_ioctl($) { + my ($ioctl_nr) = @_; + my $size = ($ioctl_nr >> 16) & 0x3fff; + my $dir = ($ioctl_nr >> 30); + my $base = $ioctl_nr & 0x1fffffff; + die "invalid size $size" if ($size > (1 << 13)); + return $base | ($dir + 1) << 29; +} + +# +# Use FIBMAP to gather block lists, block-by-block +# This is the older, slower way to iterate over file extents but +# will generally work on older kernels. The block number also +# must fit into an integer limiting the size of the file system. +# +sub fibmap_blocklist($$$) { + my ($fd, $filesize, $bsize) = @_; + + my $exts = ''; + my $blocks = int(($filesize + $bsize - 1) / $bsize); + die("file is too big for fibmap\n") if $blocks * $bsize < $filesize; + my ($firstblock, $lastblock); + for (my $cnt = 0; $cnt < $blocks; ++$cnt) { + my $block = pack('I', $cnt); + if (not defined ioctl($fd, $FIBMAP, $block)) { + if (not defined ioctl($fd, alt_ioctl($FIBMAP), $block)) { + die("fibmap ioctl: $!\n"); + } + } + $block = unpack('I', $block); + if (defined($firstblock) && $block == ($firstblock ? $lastblock + 1 : 0)) { + $lastblock++; # continue run + next; + } + # finish old run + $exts .= "-$lastblock" if defined($firstblock) && $firstblock != $lastblock; + # start new run + $exts .= " $block"; + $firstblock = $lastblock = $block; + } + # finish last run + $exts .= "-$lastblock" if defined($firstblock) && $firstblock != $lastblock; + return $exts; +} + +# +# Use the FIEMAP ioctl to gather block lists, defined extent at a time +# This is the newer way to gather extent information. We iterate the file +# up to 50 extents at a time, each describing a contiguous, non-hole, range. +# +# see /usr/include/linux/fiemap.h for definitions of the flags used below +# +my $q_emu = 0; # 0=no emu, 1=little endian, 2=big endian + +sub pack_Q { + my ($v) = @_; + return pack('Q', $v) unless $q_emu; + my $v1 = int($v / 4294967296); + my $v2 = int($v - 4294967296 * $v1); + ($v1, $v2) = ($v2, $v1) if $q_emu == 1; + return pack('LL', $v1, $v2); +} + +sub unpack_Q { + return unpack('Q', $_[0]) unless $q_emu; + my ($v1, $v2) = unpack('LL', $_[0]); + ($v1, $v2) = ($v2, $v1) if $q_emu == 1; + my $v = $v1 * 4294967296 + $v2; + die("unpack_Q: number too big\n") if int($v - 4294967296 * $v1) != $v2; + return $v; +} + +# Convert a size into a block number and an offset. +sub bytes_in_blocks($$) { + my ($bytes, $bsize) = @_; + my $blk = int($bytes / ($bsize * 65536)) * 65536; # work with 32bit int + $blk += int(($bytes - $blk * $bsize) / $bsize); + return ($blk, $bytes - $blk * $bsize); +} + +# Create an extent descriptor +sub createext($$$;$) { + my ($start, $end, $bsize, $islast) = @_; + die if $start > $end; + my ($startblk, $startoff) = bytes_in_blocks($start, $bsize); + my ($endblk, $endoff) = bytes_in_blocks($end, $bsize); + $endoff = $bsize - 1 if $islast; + my $ext = " $startblk"; + $ext .= ":$startoff" if $startoff != 0; + if ($startblk != $endblk || $endoff != $bsize - 1) { + $ext .= "-$endblk"; + $ext .= ":$endoff" if $endoff != $bsize - 1; + } + return $ext; +} + +sub fiemap_blocklist($$$) { + my ($file, $filesize, $bsize) = @_; + + # check if pack/unpack supports the Q template + $q_emu = 0; + eval { die if unpack('Q', pack('Q', 1)) != 1 }; + if ($@) { + # nope, fallback to encode/decode helpers + $q_emu = unpack('C', pack('L', 1)) ? 1 : 2; + } + + my $exts = ''; + my $offset = 0; + while ($offset < $filesize) { + my $flags_in = 0x00000001; # FIEMAP_FLAG_SYNC + my $x = pack("a8a8IIIx4.", pack_Q($offset), pack_Q($filesize), $flags_in, 0, 50, 4096); + + if (not defined ioctl($file, $FIEMAP, $x)) { + if (not defined ioctl($file, alt_ioctl($FIEMAP), $x)) { + die("fiemap ioctl: $!\n"); + } + } + + my ($flags, $count, @extents) = unpack('x16IIx8(a8a8a8a8a8IIII)[50]', $x); + last if $count == 0; # rest is a hole + + for (my $i = 0; $i < $count; $i++) { + $extents[$_] = unpack_Q($extents[$_]) for 0, 1, 2; + my ($logical, $physical, $length, $resv1, $resv2, $flags) = splice(@extents, 0, 9); + die("logical offset outside of file?\n") if $logical < 0 || $logical >= $filesize; + die("going back in file?\n") if $offset > $logical; + die("extent with bad size?\n") if $length <= 0; + # add a hole entry if needed + $exts .= createext(0, $logical - $offset - 1, $bsize) if $offset < $logical; + my $islast = $logical + $length >= $filesize ? 1 : 0; + # Not a hole but for these purposes we should treat it as one + if ($flags & 0x00000800) { # UNWRITTEN + $exts .= createext(0, $length - 1, $bsize, $islast); + } elsif ($flags & 0x00000008) { # ENCODED + die("extent mapped but is encoded\n"); + # UNKNOWN|DELALLOC|DATA_ENCRYPTED|NOT_ALIGNED|DATA_INLINE|DATA_TAIL + } elsif ($flags & 0x00000786) { + die("extent cannot be block-mapped\n"); + } else { + $exts .= createext($physical, $physical + $length - 1, $bsize, $islast); + } + $offset = $logical + $length; + } + } + $exts .= createext(0, $filesize - $offset - 1, $bsize, 1) if $offset < $filesize; + return $exts; +} + +my ($opt_padstart, $opt_padend, $opt_verbose, $opt_manifest, $opt_mani0, $opt_fibmap); $opt_verbose = 0; while (@ARGV) { @@ -62,6 +218,11 @@ $opt_manifest = shift @ARGV; next; } + if ($ARGV[0] eq '--fibmap') { + shift @ARGV; + $opt_fibmap = 1; + next; + } last; } @@ -100,18 +261,18 @@ my $c = readlink($file); die("$file: readlink: $!\n") unless defined $c; if ("/$c/" =~ /\/\.?\//s) { - print STDERR "$file: bad symlink ($c) ignored\n"; + print STDERR "$file: bad symlink ($c), ignored\n"; next; } if ("/$c/" =~ /^((?:\/\.\.)+)\/(.*?)$/s) { my ($head, $tail) = ($1, $2); if (("/$tail/" =~ /\/\.\.\//s) || (($head =~ y!/!!) > ($file =~ y!/!!))) { - print STDERR "$file: bad symlink ($c) ignored\n"; + print STDERR "$file: bad symlink ($c), ignored\n"; next; } } else { if ("/$c/" =~ /\/\.\.\//s) { - print STDERR "$file: bad symlink ($c) ignored\n"; + print STDERR "$file: bad symlink ($c), ignored\n"; next; } } @@ -119,68 +280,51 @@ print "l $n $c\n"; next; } elsif (-d _) { - print STDERR "$file\n" if $opt_verbose && $opt_verbose > 1; + print STDERR "$file\n" if $opt_verbose && ($opt_verbose > 1 || $file =~ /^KIWI\/[^\/]*$/); print "d $n\n"; next; - } elsif (!-f _) { + } elsif (! -f _) { + print STDERR "$file: unsupported file type, ignored\n"; next; } print STDERR "$file\n" if $opt_verbose && !($opt_verbose == 1 && $file =~ /^KIWI\/.*\//); - if (!open(F, '<', $file)) { - print STDERR "$file: $!"; + my $fd = undef; + if (!open($fd, '<', $file)) { + print STDERR "$file: $!\n"; next; } - my @stat = stat(F); + my @stat = stat($fd); die unless @stat; - my $st_size = $stat[7]; - if ($st_size == 0) { + my $filesize = $stat[7]; + if ($filesize == 0) { print "f $n 0\n"; - close F; + close($fd); next; } - my $bsize = 'xxxx'; - ioctl(F, 2, $bsize) || ioctl(F, 536870914, $bsize) || die("FIGETBSZ: $!\n"); - $bsize = unpack("L", $bsize); - die("$file: empty blocksize\n") unless $bsize != 0; - - print "f $n $st_size $bsize"; - my $blocks = int(($st_size+$bsize-1)/$bsize); - my ($firstblock, $lastblock); - for ($b = 0; $b < $blocks; ++$b) { - my $block = pack('I', $b); - if (not defined ioctl(F, 1, $block)) { - if (not defined ioctl(F, 536870913, $block)) { - die "$file: $!\n"; - } + my $bsize = pack('I', 0); + if (not defined ioctl($fd, $FIGETBSZ, $bsize)) { + if (not defined ioctl($fd, alt_ioctl($FIGETBSZ), $bsize)) { + die("$file: FIGETBSZ failed: $!\n"); } - $block = unpack('I', $block); - if (!$firstblock && defined($firstblock)) { - # last block was hole - if (!$block) { - $lastblock++; # count holes, 0-2 means three hole blocks - } else { - # switch back from 'hole mode' into normal mode - printf "-$lastblock" if defined($firstblock) && $firstblock != $lastblock; - print " $block"; - $firstblock = $lastblock = $block; - } - next; - } - if (!$firstblock || $lastblock + 1 != $block) { - # start of a new run - printf "-$lastblock" if defined($firstblock) && $firstblock != $lastblock; - print " $block"; - $firstblock = $block; - } - $lastblock = $block; } - # finish last run - printf "-$lastblock" if defined($firstblock) && $firstblock != $lastblock; - close F; - print "\n"; + $bsize = unpack('I', $bsize); + die("$file: illegal blocksize $bsize\n") unless $bsize > 0; + + my $exts; + if (!$opt_fibmap) { + eval { $exts = fiemap_blocklist($fd, $filesize, $bsize) }; + warn($@) if $@; + } + if (!defined($exts)) { + eval { $exts = fibmap_blocklist($fd, $filesize, $bsize) }; + warn($@) if $@; + } + die "$file: could not get extent list\n" unless defined $exts; + print "f $n $filesize $bsize$exts\n"; + close($fd); } print "\n"x$opt_padend if $opt_padend; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/configs/sles15.conf new/obs-build-20180629/configs/sles15.conf --- old/obs-build-20180329/configs/sles15.conf 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/configs/sles15.conf 2018-06-29 07:58:19.000000000 +0200 @@ -49,9 +49,6 @@ # Build all packages with -pie enabled Required: gcc-PIE -# needed for su's default config - perhaps we should use a simplified form? -Support: pam-modules - # the basic stuff Support: perl Support: brp-check-suse post-build-checks rpmlint-Factory diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/dist/build.changes new/obs-build-20180629/dist/build.changes --- old/obs-build-20180329/dist/build.changes 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/dist/build.changes 2018-06-29 07:58:19.000000000 +0200 @@ -1,4 +1,14 @@ ------------------------------------------------------------------- +Tue Apr 3 11:30:47 UTC 2018 - [email protected] + +- KIWI: Support OBS-Profiles kiwi annotation + kiwi profile filtering +- Switch to a little perl http server to get rid of the python2 dependency +- Avoid Ubuntu 18.04 uncompress failure messages +- Don't rely on AppImage's auto-detection for architecture +- build-recipe-livebuild: count ONIE images as build results + +------------------------------------------------------------------- Fri Feb 9 15:17:38 UTC 2018 - [email protected] - add sysrq functionality for KVM builds diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/dummyhttpserver new/obs-build-20180629/dummyhttpserver --- old/obs-build-20180329/dummyhttpserver 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/dummyhttpserver 2018-06-29 07:58:19.000000000 +0200 @@ -108,8 +108,8 @@ while (1) { next if $path =~ s!//!/!; next if $path =~ s!/\.(?:/|$)!/!; - next if $path =~ s!/[^/]+/..(?:/|$)!/!; - next if $path =~ s!/..(?:/|$)!/!; + next if $path =~ s!/[^/]+/\.\.(?:/|$)!/!; + next if $path =~ s!/\.\.(?:/|$)!/!; last; } return ($path, $query_string, $qu); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/expanddeps new/obs-build-20180629/expanddeps --- old/obs-build-20180329/expanddeps 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/expanddeps 2018-06-29 07:58:19.000000000 +0200 @@ -423,6 +423,11 @@ my @sysdeps = Build::get_sysbuild($cf, $buildtype, $extrasysdeps); if ($buildtype eq 'kiwi-image' || $buildtype eq 'kiwi-product') { + if (!shift @sysdeps) { + print STDERR "expansion error\n"; + print STDERR " $_\n" for @sysdeps; + exit(1); + } # just use the sysdeps for now, ignore real deps print_rpmlist(@sysdeps); exit(0); @@ -450,8 +455,18 @@ @bdeps = Build::unify(@sysdeps, @bdeps); } -# make sure all preinstalls are in bdeps; -@bdeps = Build::unify(@bdeps, Build::get_preinstalls($cf)); -@bdeps = Build::unify(@bdeps, Build::get_vminstalls($cf)) if $isvm; +# get preinstalls/vminstalls and check if the packages exist +my @xdeps = Build::get_preinstalls($cf); +push @xdeps, Build::get_vminstalls($cf) if $isvm; +my @missing = grep {!$packs{$_}} @xdeps; +if (@missing) { + @missing = sort(Build::unify(@missing)); + print STDERR "expansion error\n"; + print STDERR " missing pre/vminstalls: ".join(', ', @missing)."\n"; + exit(1); +} + +# make sure all preinstalls/vminstalls are in bdeps +@bdeps = Build::unify(@bdeps, @xdeps); print_rpmlist(@bdeps); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/extractbuild new/obs-build-20180629/extractbuild --- old/obs-build-20180329/extractbuild 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/extractbuild 2018-06-29 07:58:19.000000000 +0200 @@ -2,7 +2,7 @@ ################################################################ # -# Copyright (c) 1995-2014 SUSE Linux Products GmbH +# Copyright (c) 1995-2018 SUSE Linux Products GmbH # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or 3 as @@ -71,7 +71,7 @@ while (<S>) { chomp; last unless length $_; - my ($filetype, $file, $filesize, $blksize, @blocks) = split(/ /); + my ($filetype, $file, $filesize, $blksize, @blocks) = split(' '); die("invalid input '$_'\n") unless defined($file); $file =~ s/%([a-fA-F0-9]{2})/chr(hex($1))/ge; die("bad file '$file' (contains \\0)\n") if $file =~ /\0/; @@ -81,7 +81,7 @@ die("file without directory: $file\n") unless $done{$1} && $done{$1} eq 'd'; } if ($filetype eq 'd') { # dir - print "$file\n" if $opt_verbose && $opt_verbose > 1; + print "$file\n" if $opt_verbose && ($opt_verbose > 1 || $file =~ /^KIWI\/[^\/]*$/); mkdir($file) || die("mkdir $file: $!\n"); $done{$file} = 'd'; next; @@ -105,54 +105,46 @@ next; } die("illegal file type: $filetype\n") unless $filetype eq 'f'; - die "invalid input '$_'\n" if !@blocks && $filesize; + print "$file\n" if $opt_verbose && !($opt_verbose == 1 && $file =~ /^KIWI\/.*\//); $done{$file} = 'f'; - $filesize = int($filesize); + open (O, '>', $file) or die "$file: $!\n"; if ($filesize == 0) { - print "$file\n" if $opt_verbose && !($opt_verbose == 1 && $file =~ /^KIWI\/.*\//); - open (O, '>', $file) or die "$file: $!\n"; - close O; + close(O) || die("$file: close error: $!\n"); next; } $blksize = int($blksize); die "$file: invalid block size $blksize\n" unless $blksize > 0 && $blksize <= $bufsize; - my $maxblocks = int($bufsize/$blksize); - print "$file\n" if $opt_verbose && !($opt_verbose == 1 && $file =~ /^KIWI\/.*\//); - open (O, '>', $file) or die "$file: $!\n"; + my $needtruncate; + my $left = $filesize; for my $block (@blocks) { - my $end; - ($block, $end) = split(/-/, $block); - $block = int($block); - if ($block == 0) { # a hole! - $end = (($end || 0) + 1) * $blksize; - $end = $filesize if $end > $filesize; - seek(O, $end, 1); - $filesize -= $end; + die("bad extent '$block'\n") unless $block =~ /^(\d+)(?::(\d+))?(?:-(\d+)(?::(\d+))?)?$/; + my ($startblk, $startoff, $endblk, $endoff) = ($1, $2, $3, $4); + $startoff = 0 unless defined $startoff; + $endblk = $startblk unless defined $endblk; + $endoff = $blksize - 1 unless defined $endoff; + my $start = $startblk * $blksize + $startoff; + my $len = $endblk * $blksize + $endoff + 1 - $start; + die "$file: bad length\n" if $len <= 0; + die "$file: extent is outside of file\n" if $left <= 0; + $len = $left if $len > $left; # it's ok to overshoot the last block + $left -= $len; + if ($start == 0) { # a hole! + seek(O, $len, 1); + $needtruncate = 1; next; } - $end = $block unless $end; - $end = int($end); - seek(F, $block*$blksize, 0) || die "$file: seek: $!\n"; - while ($block <= $end && $filesize) { - my $size; - if ($end == $block) { - $size = $blksize; - ++$block; - } elsif ($maxblocks >= $end-$block) { - $size = ($end-$block)*$blksize; - $block += $end-$block; - } else { - $size = $maxblocks*$blksize; - $block += $maxblocks; - } - $size = $filesize if $size > $filesize; + $needtruncate = undef; + seek(F, $start, 0) || die "$file: seek: $!\n"; + while ($len > 0) { + my $size = $len > $bufsize ? $bufsize : $len; my $buf; (sysread(F, $buf, $size) || 0) == $size || die("$file: read: $!\n"); - $filesize -= $size; (syswrite(O, $buf) || 0) == length($buf) || die("$file: write error\n"); + $len -= $size; } } + truncate(O, $filesize) if $needtruncate; close(O) || die("$file: close error: $!\n"); # sanity check - die "$file: invalid file size ($filesize bytes left)\n" if $filesize != 0; + die "$file: invalid file size ($left bytes left)\n" if $left != 0; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20180329/init_buildsystem new/obs-build-20180629/init_buildsystem --- old/obs-build-20180329/init_buildsystem 2018-03-29 17:01:40.000000000 +0200 +++ new/obs-build-20180629/init_buildsystem 2018-06-29 07:58:19.000000000 +0200 @@ -356,7 +356,7 @@ REPOTYPE=rpm-md ;; *.dsc) REPOTYPE=debian ;; - */PKGBUILD|PKGBUILD) + */PKGBUILD|PKGBUILD|*/_service:*:PKGBUILD|_service:*:PKGBUILD) REPOTYPE=arch ;; esac fi
