Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package build for openSUSE:Factory checked in at 2023-08-10 15:33:04 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/build (Old) and /work/SRC/openSUSE:Factory/.build.new.11712 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "build" Thu Aug 10 15:33:04 2023 rev:155 rq:1102975 version:20230808 Changes: -------- --- /work/SRC/openSUSE:Factory/build/build.changes 2023-06-29 17:27:38.426017537 +0200 +++ /work/SRC/openSUSE:Factory/.build.new.11712/build.changes 2023-08-10 15:33:11.251896993 +0200 @@ -1,0 +2,13 @@ +Tue Aug 8 15:30:05 UTC 2023 - Adrian Schröter <adr...@suse.de> + +- pbuild: + * fix dependency expansion for build types other than spec + * Reworked cycle handling code + * add --extra-packs option + * add debugflags option +- Pass-through --buildtool-opt +- Parse Patch and Source lines more accurately +- fix tunefs functionality +- minor bugfixes + +------------------------------------------------------------------- Old: ---- obs-build-20230628.tar.gz New: ---- obs-build-20230808.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ build.spec ++++++ --- /var/tmp/diff_new_pack.u09UZM/_old 2023-08-10 15:33:12.183902805 +0200 +++ /var/tmp/diff_new_pack.u09UZM/_new 2023-08-10 15:33:12.187902830 +0200 @@ -28,7 +28,7 @@ Summary: A Script to Build SUSE Linux RPMs License: GPL-2.0-only OR GPL-3.0-only Group: Development/Tools/Building -Version: 20230628 +Version: 20230808 Release: 0 Source: obs-build-%{version}.tar.gz BuildRoot: %{_tmppath}/%{name}-%{version}-build ++++++ PKGBUILD ++++++ --- /var/tmp/diff_new_pack.u09UZM/_old 2023-08-10 15:33:12.243903179 +0200 +++ /var/tmp/diff_new_pack.u09UZM/_new 2023-08-10 15:33:12.247903205 +0200 @@ -1,5 +1,5 @@ pkgname=build -pkgver=20230628 +pkgver=20230808 pkgrel=0 pkgdesc="Build packages in sandbox" arch=('i686' 'x86_64') ++++++ _service ++++++ --- /var/tmp/diff_new_pack.u09UZM/_old 2023-08-10 15:33:12.279903404 +0200 +++ /var/tmp/diff_new_pack.u09UZM/_new 2023-08-10 15:33:12.279903404 +0200 @@ -1,7 +1,7 @@ <services> <service name="tar_scm" mode="manual"> - <param name="revision">20230628</param> - <param name="version">20230628</param> + <param name="revision">20230808</param> + <param name="version">20230808</param> <param name="url">https://github.com/openSUSE/obs-build.git</param> <param name="scm">git</param> <param name="extract">dist/build.changes</param> ++++++ build.dsc ++++++ --- /var/tmp/diff_new_pack.u09UZM/_old 2023-08-10 15:33:12.307903578 +0200 +++ /var/tmp/diff_new_pack.u09UZM/_new 2023-08-10 15:33:12.311903604 +0200 @@ -1,6 +1,6 @@ Format: 1.0 Source: build -Version: 20230628 +Version: 20230808 Binary: build Maintainer: Adrian Schroeter <adr...@suse.de> Architecture: all ++++++ debian.changelog ++++++ --- /var/tmp/diff_new_pack.u09UZM/_old 2023-08-10 15:33:12.351903853 +0200 +++ /var/tmp/diff_new_pack.u09UZM/_new 2023-08-10 15:33:12.355903878 +0200 @@ -1,4 +1,4 @@ -build (20230628) unstable; urgency=low +build (20230808) unstable; urgency=low * Update to current git trunk - add sles11sp2 build config and adapt autodetection ++++++ obs-build-20230628.tar.gz -> obs-build-20230808.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/Build/Expand.pm new/obs-build-20230808/Build/Expand.pm --- old/obs-build-20230628/Build/Expand.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/Build/Expand.pm 2023-08-08 17:34:27.000000000 +0200 @@ -375,6 +375,7 @@ sub expand { my ($config, @p) = @_; + print "expand: @p\n" if $expand_dbg; my $conflicts = $config->{'conflicth'}; my $pkgconflicts = $config->{'pkgconflictsh'} || {}; my $pkgobsoletes = $config->{'pkgobsoletesh'} || {}; @@ -698,7 +699,7 @@ for my $q (@q) { for my $rs (@{$pkgsupplements->{$q} || []}) { if ($rs =~ /^\(.*\)$/) { - my $rd = Build::Rpm::parse_rich_dep($rs); + my $rd = Build::Rpm::parse_rich_dep($rs); next if !$rd || fulfilled_cplx_rec($config, \%p, $rd); } else { next unless grep {$p{$_}} @{$whatprovides->{$rs} || Build::addproviders($config, $rs)}; @@ -790,7 +791,7 @@ for my $r (splice @native) { next if $rdone{$r}++; if ($r eq '--directdepsend--') { - push @native, $r; + push @native, $r; next; } my @q = @{$whatprovides->{$r} || Build::addproviders($config, $r)}; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/Build/Rpm.pm new/obs-build-20230808/Build/Rpm.pm --- old/obs-build-20230628/Build/Rpm.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/Build/Rpm.pm 2023-08-08 17:34:27.000000000 +0200 @@ -371,7 +371,7 @@ reexpand: while ($line =~ /^(.*?)%(\{([^\}]+)\}|[\?\!]*[0-9a-zA-Z_]+|%|\*\*?|#|\(|\[)(.*?)\z/s) { if ($tries++ > 1000) { - do_warn($config, "spec file parser ",($lineno?" line $lineno":''),": macro too deeply nested"); + do_warn($config, "spec file parser".($lineno?" line $lineno":'').": macro too deeply nested"); $line = 'MACRO'; last; } @@ -411,7 +411,7 @@ } $macname =~ s/^[\!\?]+//s; if ($macname eq '(') { - do_warn($config, "spec file parser",($lineno?" line $lineno":''),": can't expand %(...)"); + do_warn($config, "spec file parser".($lineno?" line $lineno":'').": can't expand %(...)"); $line = 'MACRO'; last; } elsif ($macname eq '[') { @@ -485,7 +485,7 @@ $expandedline .= luamacro($config, $macname, @args); } elsif (exists($macros->{$macname})) { if (!defined($macros->{$macname})) { - do_warn($config, "spec file parser",($lineno?" line $lineno":''),": can't expand '$macname'"); + do_warn($config, "spec file parser".($lineno?" line $lineno":'').": can't expand '$macname'"); $line = 'MACRO'; last; } @@ -637,7 +637,7 @@ $ret->{'error'} = "open $specfile: $!"; return $ret; } - + initmacros($config, \%macros, \%macros_args); my $skip = 0; my $main_preamble = 1; @@ -923,32 +923,33 @@ } } next; - } elsif ($preamble && $line =~ /^(Source\d*|Patch\d*|Url|Icon)\s*:\s*(\S+)/i) { + } elsif ($preamble && $line =~ /^(Url|Icon)\s*:\s*(\S+)/i) { my ($tag, $val) = (lc($1), $2); $macros{$tag} = $val if $tag eq 'url'; # associate url and icon tags with the corresponding subpackage - $tag .= scalar @subpacks if ($tag eq 'url' || $tag eq 'icon') && @subpacks; + $tag .= scalar @subpacks if @subpacks; if ($tag =~ /icon/) { # there can be a gif and xpm icon push @{$ret->{$tag}}, $val; } else { - if ($tag =~ /^(source|patch)(\d+)?$/) { - my $num = defined($2) ? 0 + $2 : $autonum{$1}; - $tag = "$1$num"; - if ($tag eq 'patch0' && exists($ret->{$tag})) { - # gross hack. Before autonumbering "Patch" and "Patch0" could - # exist. So take out the previous patch and add it back - # without number. This does not exactly work as old rpms - # but hopefully good enough :-) - $ret->{'patch'} = delete $ret->{$tag}; - } - do_warn($config, "spec file parser: $tag already exists") if exists $ret->{$tag}; - $autonum{$1} = $num + 1 if $num >= $autonum{$1}; - $macros{uc($1) . "URL$num"} = $val; - } $ret->{$tag} = $val; } - if ($remoteasset && $tag =~ /^(?:source|patch)/) { + } elsif ($preamble && $line =~ /^(Source|Patch)(\d*)\s*(\s+[^:]*?)?:\s*(\S+)/i) { + my ($tagtype, $num, $tagextra, $val) = (lc($1), $2, $3, $4); + $num = $num ne '' ? 0 + $num : $autonum{$tagtype}; + my $tag = "$tagtype$num"; + if ($tag eq 'patch0' && exists($ret->{$tag})) { + # gross hack. Before autonumbering "Patch" and "Patch0" could + # exist. So take out the previous patch and add it back + # without number. This does not exactly work as old rpms + # but hopefully good enough :-) + $ret->{'patch'} = delete $ret->{$tag}; + } + do_warn($config, "spec file parser: $tag already exists") if exists $ret->{$tag}; + $autonum{$tagtype} = $num + 1 if $num >= $autonum{$tagtype}; + $macros{uc($tagtype) . "URL$num"} = $val; + $ret->{$tag} = $val; + if ($remoteasset) { $remoteasset->{'url'} = $val; push @{$ret->{'remoteassets'}}, $remoteasset; } @@ -1570,7 +1571,7 @@ die("$path: no md5 signature header\n") unless $idxarea =~ /\A(?:.{16})*\000\000\003\354\000\000\000\007(....)\000\000\000\020/s; my $md5off = unpack('N', $1); die("$path: bad md5 offset\n") unless $md5off; - $md5off += 96 + 16 + $cnt * 16; + $md5off += 96 + 16 + $cnt * 16; $hdrmd5 = unpack("\@${md5off}H32", $buf); } ($headmagic, $cnt, $cntdata) = unpack('N@8NN', substr($buf, $hlen)); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/PBuild/Checker.pm new/obs-build-20230808/PBuild/Checker.pm --- old/obs-build-20230628/PBuild/Checker.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/PBuild/Checker.pm 2023-08-08 17:34:27.000000000 +0200 @@ -56,6 +56,7 @@ 'assetmgr' => $assetmgr, }; $ctx->{'rebuild'} = $opts->{'buildtrigger'} if $opts->{'buildtrigger'}; + $Build::Expand::expand_dbg = 1 if $opts->{'debugflags'}->{'expansion'}; return bless $ctx; } @@ -88,7 +89,9 @@ my ($ctx, @pkgs) = @_; my $bconf = $ctx->{'bconf'}; my $bconf_host = $ctx->{'bconf_host'}; + my $expand_dbg = $Build::Expand::expand_dbg; if (($bconf_host || $bconf)->{'expandflags:preinstallexpand'}) { + print "=== preinstall expansion\n" if $expand_dbg; my $err = Build::expandpreinstalls($bconf_host || $bconf); die("cannot expand preinstalls: $err\n") if $err; } @@ -96,6 +99,7 @@ my $subpacks = $ctx->{'subpacks'}; my $cross = $bconf_host ? 1 : 0; for my $pkg (@pkgs) { + print "=== dependency expansion of $pkg\n" if $expand_dbg; my $p = $pkgsrc->{$pkg}; if ($p->{'native'}) { PBuild::Expand::expand_deps($p, $bconf_host, $subpacks); @@ -106,6 +110,30 @@ } # +# Sort the components of a dependency cycle +# +sub cycsort { + my ($pkg2dep, $dep2src, $pkg2src, @cyc) = @_; + + @cyc = sort(@cyc); + my %d; + my %cdeps; + for my $pkg (@cyc) { + $d{$dep2src->{$_} || $_}->{$pkg} = 1 for @{$pkg2dep->{$pkg}}; + } + # remove all bi-directional edges + my %ign; + for my $pkg (@cyc) { + $ign{$pkg}->{$_} = 1 for keys %{$d{$pkg2src->{$pkg}} || {}}; + } + for my $pkg (@cyc) { + $_ ne $pkg && !$ign{$_}->{$pkg} and push @{$cdeps{$_}}, $pkg for keys %{$d{$pkg2src->{$pkg}} || {}}; + } + @cyc = PBuild::Depsort::depsort(\%cdeps, undef, undef, undef, @cyc); + return @cyc; +} + +# # Sort the packages by dependencies # sub pkgsort { @@ -115,7 +143,7 @@ my $pkgsrc = $ctx->{'pkgsrc'}; for my $pkg (@pkgs) { my $p = $pkgsrc->{$pkg}; - $pdeps{$pkg} = $p->{'dep_expanded'} || []; + $pdeps{$pkg} = $p->{'dep_expanded'} || []; $pkg2src{$pkg} = $p->{'name'} || $p->{'pkg'}; } my @cycles; @@ -125,8 +153,8 @@ for my $cyc (@sccs) { next if @$cyc < 2; # just in case my @c = map {@{$cychash{$_} || [ $_ ]}} @$cyc; - @c = PBuild::Util::unify(sort(@c)); - $cychash{$_} = \@c for @c; + @c = cycsort(\%pdeps, $ctx->{'dep2src'}, \%pkg2src, @c); + $cychash{$_} = \@c for @c; } #if (@sccs) { # print " sccs:\n"; @@ -155,6 +183,7 @@ $ctx->{'building'}->{$_->{'job'}->{'pdata'}->{'pkg'}} = $_->{'job'} for grep {$_->{'job'}} @$builders; $ctx->{'notready'} = {}; # building or blocked $ctx->{'nharder'} = 0; + $ctx->{'cyclevel'} = {}; my $builddir = $ctx->{'builddir'}; my $pkgsrc = $ctx->{'pkgsrc'}; @@ -167,10 +196,11 @@ my $packid = shift @cpacks; # cycle handling code - my $incycle = 0; + my $incycle = 0; if ($cychash->{$packid}) { - ($packid, $incycle) = handlecycle($ctx, $packid, \@cpacks, \%cycpass, \%packstatus); - next unless $incycle; + ($packid, $incycle) = handlecycle($ctx, $packid, \@cpacks, \%cycpass); + next if $incycle == 4; # ignore after pass1/2 + next if $packstatus{$packid} && $packstatus{$packid} ne 'done' && $packstatus{$packid} ne 'succeeded' && $packstatus{$packid} ne 'failed'; # already decided } my $p = $pkgsrc->{$packid}; if ($p->{'error'}) { @@ -194,6 +224,7 @@ $packstatus{$packid} = 'building'; $packdetails{$packid} = "on builder $job->{'name'}" if $job->{'nbuilders'} > 1; $ctx->{'notready'}->{$p->{'name'} || $p->{'pkg'}} = 1 if $p->{'useforbuildenabled'}; + $ctx->{'cyclevel'}->{$packid} = $job->{'cyclevel'} if $incycle && $job->{'cyclevel'}; next; } @@ -224,6 +255,7 @@ print "${bid}building $p->{'pkg'}/$p->{'recipe'}\n"; } $ctx->{'building'}->{$packid} = $builder->{'job'}; + $job->{'cyclevel'} = $ctx->{'cyclevel'}->{$packid} if $incycle && $ctx->{'cyclevel'}->{$packid}; } #printf("%s -> %s%s", $packid, $status, $error ? " ($error)" : ''); } elsif ($status eq 'done') { @@ -380,10 +412,14 @@ return ('blocked', join(', ', @blocked)); } # prune cycle packages from blocked - if ($incycle) { + if ($incycle > 1) { + my $cyclevel = $ctx->{'cyclevel'}; my $pkgsrc = $ctx->{'pkgsrc'}; - my %cycs = map {(($pkgsrc->{$_} || {})->{'name'} || $_) => 1} @{$ctx->{'cychash'}->{$packid}}; - @blocked = grep {!$cycs{$dep2src->{$_}}} @blocked; + my $level = $cyclevel->{$packid}; + if ($level) { + my %cycs = map {(($pkgsrc->{$_} || {})->{'name'} || $_) => ($cyclevel->{$_} || 1)} @{$ctx->{'cychash'}->{$packid}}; + @blocked = grep {($cycs{$dep2src->{$_}} || 0) < $level} @blocked; + } } if (@blocked) { # print " - $packid ($buildtype)\n"; @@ -449,11 +485,6 @@ goto relsynccheck; } # more work, check if dep rpm changed - if ($incycle == 1) { - # print " - $packid ($buildtype)\n"; - # print " in cycle, no source change...\n"; - return ('done'); - } my $check = substr($mylastcheck, 32, 32); # metamd5 my $dep2pkg = $p->{'native'} ? $ctx->{'dep2pkg_host'} : $ctx->{'dep2pkg'}; @@ -484,6 +515,16 @@ close F; chomp @meta; } + if ($incycle == 1) { + # calculate cyclevel + my $level = PBuild::Meta::diffdepth(\@meta, $new_meta); + $ctx->{'cyclevel'}->{$packid} = $level; + if ($level > 1) { + # print " - $packid ($buildtype)\n"; + # print " in cycle, no source change...\n"; + return ('done'); # postpone till phase 2 + } + } if ($rebuildmethod eq 'direct') { @meta = grep {!/\//} @meta; @$new_meta = grep {!/\//} @$new_meta; @@ -515,46 +556,28 @@ # Build dependency cycle handling # sub handlecycle { - my ($ctx, $packid, $cpacks, $cycpass, $packstatus) = @_; - - my $incycle = 0; + my ($ctx, $packid, $cpacks, $cycpass) = @_; my $cychash = $ctx->{'cychash'}; return ($packid, 0) unless $cychash->{$packid}; - # do every package in the cycle twice: - # pass1: only build source changes - # pass2: normal build, but block if a pass1 package is building - # pass3: ignore - $incycle = $cycpass->{$packid}; - if (!$incycle) { - # starting pass 1 (incycle == 1) - my @cycp = @{$cychash->{$packid}}; - unshift @$cpacks, $cycp[0]; # pass3 - unshift @$cpacks, @cycp; # pass2 - unshift @$cpacks, @cycp; # pass1 + my $incycle = $cycpass->{$packid}; + return ($packid, $incycle) if $incycle > 0; # still in pass + my @cycp = @{$cychash->{$packid}}; + $incycle = -$incycle + 1; # start next pass + $cycpass->{$_} = $incycle for @cycp; + if ($incycle == 1) { + unshift @$cpacks, $cycp[0]; + unshift @$cpacks, @cycp; + $packid = shift @$cpacks; + $cycpass->{$packid} = -1; # set pass1 endmarker + } elsif ($incycle == 2) { + my $cyclevel = $ctx->{'cyclevel'}; + unshift @$cpacks, sort {($cyclevel->{$a} || 0) <=> ($cyclevel->{$b} || 0)} @cycp; + $packid = shift @$cpacks; + $cycpass->{$packid} = -2; # set pass2 endmarker + } elsif ($incycle == 3) { + unshift @$cpacks, @cycp; $packid = shift @$cpacks; - $incycle = 1; - $cycpass->{$_} = $incycle for @cycp; - $cycpass->{$packid} = -1; # pass1 ended - } elsif ($incycle == -1) { - # starting pass 2 (incycle will be 2 or 3) - my @cycp = @{$cychash->{$packid}}; - $incycle = (grep {$ctx->{'building'}->{$_}} @cycp) ? 3 : 2; - $cycpass->{$_} = $incycle for @cycp; - $cycpass->{$packid} = -2; # pass2 ended - } elsif ($incycle == -2) { - # starting pass 3 (incycle == 4) - my @cycp = @{$cychash->{$packid}}; - $incycle = 4; - $cycpass->{$_} = $incycle for @cycp; - # propagate notready to all cycle packages - my $notready = $ctx->{'notready'}; - my $pkgsrc = $ctx->{'pkgsrc'}; - if (grep {$notready->{($pkgsrc->{$_} || {})->{'name'} || $_}} @cycp) { - $notready->{($pkgsrc->{$_} || {})->{'name'} || $_} ||= 1 for @cycp; - } } - return ($packid, undef) if $incycle == 4; # ignore after pass1/2 - return ($packid, undef) if $packstatus->{$packid} && $packstatus->{$packid} ne 'done' && $packstatus->{$packid} ne 'succeeded' && $packstatus->{$packid} ne 'failed'; # already decided return ($packid, $incycle); } @@ -592,11 +615,13 @@ my $reason = $data->[0]; my $hdeps = $data->[1]; #print Dumper($reason); + my $opts = $ctx->{'opts'}; my %jobopts; $jobopts{'nounchanged'} = 1 if $packid && $ctx->{'cychash'}->{$packid}; my @btdeps; my $edeps = $p->{'dep_expanded'} || []; - my $bconf = $ctx->{'bconf_host'} || $ctx->{'bconf'}; + my $bconf_host = $ctx->{'bconf_host'}; + my $bconf = $bconf_host || $ctx->{'bconf'}; my $buildtype = $p->{'buildtype'}; $buildtype = 'kiwi-image' if $buildtype eq 'kiwi'; my $kiwimode; @@ -614,8 +639,11 @@ } @btdeps = PBuild::Util::unify(@btdeps); } + push @btdeps, @{$opts->{'extra-packs'}} if $opts->{'extra-packs'}; + my $expand_dbg = $Build::Expand::expand_dbg; my @sysdeps = @btdeps; unshift @sysdeps, grep {/^kiwi-.*:/} @{$p->{'dep'} || []} if $buildtype eq 'kiwi-image'; + print "=== sysdeps expansion\n" if $expand_dbg; if (@sysdeps) { @sysdeps = Build::get_sysbuild($bconf, $buildtype, [ @sysdeps ]); # cannot cache... } else { @@ -628,13 +656,12 @@ unshift @bdeps, '--directdepsend--' if @bdeps; unshift @bdeps, @{$p->{'dep_native'} || []}; unshift @bdeps, @{$genbuildreqs->[1]} if $genbuildreqs; - if (!$kiwimode && $ctx->{'bconf_host'}) { + if (!$kiwimode && $bconf_host) { unshift @bdeps, @{$p->{'dep_host'} || $p->{'dep'} || []}, @btdeps; } else { unshift @bdeps, @{$p->{'dep'} || []}, @btdeps; } push @bdeps, '--ignoreignore--' if @sysdeps || $buildtype eq 'simpleimage'; - my $opts = $ctx->{'opts'}; if ($opts->{'ccache'} && ($buildtype eq 'arch' || $buildtype eq 'spec' || $buildtype eq 'dsc')) { my $opackid = $packid; $opackid = $p->{'releasename'} if $p->{'releasename'}; @@ -648,13 +675,16 @@ if ($kiwimode || $buildtype eq 'buildenv' || $buildtype eq 'preinstallimage') { @bdeps = (1, @$edeps); # reuse edeps packages, no need to expand again } else { + print "=== builddeps expansion\n" if $expand_dbg; @bdeps = Build::get_build($bconf, $ctx->{'subpacks'}->{$p->{'name'}}, @bdeps); } if (!shift(@bdeps)) { - return ('unresolvable', join(', ', @bdeps)); + my $hint = $bconf_host ? 'host:' : ''; + return ('unresolvable', $hint . join(', ', @bdeps)); } if (@sysdeps && !shift(@sysdeps)) { - return ('unresolvable', 'sysdeps:' . join(', ', @sysdeps)); + my $hint = $bconf_host ? 'host-sysdeps:' : 'sysdeps:'; + return ('unresolvable', $hint . join(', ', @sysdeps)); } my $dep2pkg = $ctx->{'dep2pkg_host'} || $ctx->{'dep2pkg'}; my @pdeps = Build::get_preinstalls($bconf); @@ -662,16 +692,17 @@ my @missing = grep {!$dep2pkg->{$_}} (@pdeps, @vmdeps); if (@missing) { my $missing = join(', ', sort(PBuild::Util::unify(@missing))); - return ('unresolvable', "missing pre/vminstalls: $missing"); + my $hint = $bconf_host ? 'host:' : ''; + return ('unresolvable', "${hint}missing pre/vminstalls: $missing"); } my $tdeps; - $tdeps = [ @$edeps ] if !$kiwimode && !$p->{'native'} && $ctx->{'bconf_host'}; + $tdeps = [ @$edeps ] if !$kiwimode && !$p->{'native'} && $bconf_host; my $oldsrcmd5 = $p->{'srcmd5'}; $ctx->{'assetmgr'}->getremoteassets($p); return ('recheck', 'assets changed') if $p->{'srcmd5'} ne $oldsrcmd5; return ('broken', $p->{'error'}) if $p->{'error'}; # missing assets my $bins; - if ($kiwimode && $ctx->{'bconf_host'}) { + if ($kiwimode && $bconf_host) { $bins = dep2bins_host($ctx, PBuild::Util::unify(@pdeps, @vmdeps, @sysdeps)); push @$bins, @{dep2bins($ctx, PBuild::Util::unify(@bdeps))}; } else { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/PBuild/Expand.pm new/obs-build-20230808/PBuild/Expand.pm --- old/obs-build-20230628/PBuild/Expand.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/PBuild/Expand.pm 2023-08-08 17:34:27.000000000 +0200 @@ -90,6 +90,7 @@ } my @deps = @{$p->{'dep'} || []}; push @deps, '--ignoreignore--' unless ($p->{'buildtype'} || '') eq 'preinstallimage'; + local $bconf->{'type'} = $p->{'buildtype'}; my ($ok, @edeps) = Build::get_build($bconf, [], @deps); if (!$ok) { delete $p->{'dep_expanded'}; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/PBuild/Meta.pm new/obs-build-20230808/PBuild/Meta.pm --- old/obs-build-20230628/PBuild/Meta.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/PBuild/Meta.pm 2023-08-08 17:34:27.000000000 +0200 @@ -161,4 +161,33 @@ return @ret; } +# return the depth where the two metas differ +# 0: same metas +# 1: first line (i.e. different source) +# >=2: number of '/' + 1 +sub diffdepth { + my ($m1, $m2) = @_; + my $i = -1; + for (@$m1) { + $i++; + my $m = $m2->[$i]; + if (!defined($m)) { + return 1 unless $i; + my $i1 = $_ =~ y!/!/!; + return $i1 + 2; + } elsif ($m ne $_) { + return 1 unless $i; + my $i1 = $_ =~ y!/!/!; + my $i2 = $m =~ y!/!/!; + return $i1 < $i2 ? $i1 + 2 : $i2 + 2; + } + } + $i++; + my $m = $m2->[$i]; + return 0 unless defined $m; + return 1 unless $i; + my $i2 = $m =~ y!/!/!; + return $i2 + 2; +} + 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/PBuild/Options.pm new/obs-build-20230808/PBuild/Options.pm --- old/obs-build-20230628/PBuild/Options.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/PBuild/Options.pm 2023-08-08 17:34:27.000000000 +0200 @@ -121,8 +121,18 @@ 'showlog' => '', 'ccache' => \&ccache_special, 'ccache-type' => '', + 'debugflags' => \&debugflags_special, + 'extra-packs' => '::', + 'extrapacks' => 'extra-packs::', + 'X' => 'extra-packs::', }; +sub debugflags_special { + my ($opts, $opt, $origopt, $args) = @_; + my $arg = Build::Options::getarg($origopt, $args); + $opts->{'debugflags'}->{$_} = 1 for split(',', $arg); +} + sub vm_type_special { my ($opts, $opt, $origopt, $args) = @_; my $arg; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/PBuild/Preset.pm new/obs-build-20230808/PBuild/Preset.pm --- old/obs-build-20230628/PBuild/Preset.pm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/PBuild/Preset.pm 2023-08-08 17:34:27.000000000 +0200 @@ -29,7 +29,7 @@ 'pbuild' => [[ 'preset' => 'name', - 'default', + 'default', 'arch', [ 'config' ], [ 'repo' ], diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/README.md new/obs-build-20230808/README.md --- old/obs-build-20230628/README.md 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/README.md 2023-08-08 17:34:27.000000000 +0200 @@ -40,8 +40,9 @@ For hardware emulation there are --- - `qemu`, - which runs a `QEMU` system emulator inside of ``KVM``. This can - be considered also secure and reproducible. + which runs a `QEMU` system emulator. This can be considered + also secure and reproducible. It requires a kernel image as + part of the build config. - The "emulator" VM can be used to run builds using any other emulator via a wrapper script. - A `QEMU` user land emulation is also possible. This would give diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/build-recipe-appimage new/obs-build-20230808/build-recipe-appimage --- old/obs-build-20230628/build-recipe-appimage 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/build-recipe-appimage 2023-08-08 17:34:27.000000000 +0200 @@ -68,14 +68,14 @@ if test "$DO_INIT" = true; then if test -n "$DEB" -a ! -d "$BUILD_ROOT/.build.binaries/dists" ; then - echo "creating debian repository metadata..." + echo "creating debian repository metadata..." createrepo_debian $BUILD_ROOT/.build.binaries ${ARCH} ${DIST} # setup /etc/apt/sources.list mkdir -p "$BUILD_ROOT/etc/apt" echo "deb [trusted=yes] file:/.build.binaries OBS main" >> "$BUILD_ROOT/etc/apt/sources.list" fi if test -z "$DEB" -a ! -d "$BUILD_ROOT/.build.binaries/repodata" ; then - echo "creating repository metadata..." + echo "creating repository metadata..." if chroot "$BUILD_ROOT" test -x /usr/bin/createrepo_c; then chroot "$BUILD_ROOT" /usr/bin/createrepo_c /.build.binaries elif chroot "$BUILD_ROOT" test -x /usr/bin/createrepo; then diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/build-recipe-docker new/obs-build-20230808/build-recipe-docker --- old/obs-build-20230628/build-recipe-docker 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/build-recipe-docker 2023-08-08 17:34:27.000000000 +0200 @@ -312,7 +312,7 @@ for format in $(queryconfig --dist "$BUILD_DIST" --configdir "$CONFIG_DIR" --archpath "$BUILD_ARCH" buildflags+ sbom | sort -u) ; do echo "Generating $format sbom file" generate_sbom --format "$format" "$TOPDIR/DOCKER/$FILENAME.tar" > "$BUILD_ROOT$TOPDIR/DOCKER/$FILENAME.${format/cyclonedx/cdx}.json" - test -s "$BUILD_ROOT$TOPDIR/DOCKER/$FILENAME.${format/cyclonedx/cdx}.json" || rm -f "$BUILD_ROOT$TOPDIR/DOCKER/$FILENAME.${format/cyclonedx/cdx}.json" + test -s "$BUILD_ROOT$TOPDIR/DOCKER/$FILENAME.${format/cyclonedx/cdx}.json" || rm -f "$BUILD_ROOT$TOPDIR/DOCKER/$FILENAME.${format/cyclonedx/cdx}.json" done # We're done. Clean up. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/build-vm new/obs-build-20230808/build-vm --- old/obs-build-20230628/build-vm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/build-vm 2023-08-08 17:34:27.000000000 +0200 @@ -448,7 +448,7 @@ var="vm_img_mkfs_${fs}"; mkfs="${!var}" var="vm_img_mkfs_extra_options_${fs}"; mkfs_extra_options="${!var}" - var="vm_img_tunefs_${fs}"; vm_img_tunefs="${!var}" + var="vm_img_tunefs_${fs}"; tunefs="${!var}" local labelopt= test "$VM_ROOTDEV" != "${VM_ROOTDEV#LABEL=}" && labelopt="-L ${VM_ROOTDEV#LABEL=}" @@ -457,17 +457,18 @@ if test "$options" = "nodirindex" && test "$fs" = "ext2" -o "$fs" = "ext3" -o "$fs" = "ext4"; then mkfs_extra_options="$mkfs_extra_options -O ^dir_index" fi + local mkfs_populate_options if test -n "$populate"; then if test "$fs" = "ext2" -o "$fs" = "ext3" -o "$fs" = "ext4"; then - mkfs_extra_options="$mkfs_extra_options -d $populate" + mkfs_populate_options="-d $populate" # mkfs needs to see the files it copies into the fs as owned by root. # Otherwise the whole fs tree would be owned by the calling user. # Doesn't matter too much when setting up the build root initially as # rpm is called inside anway to fix everything. However, when reusing # an image suddenly all ownerships would change. - [ -x "/usr/bin/unshare" ] && mkfs="unshare --user --map-root-user $mkfs" + test $UID != 0 -a -x /usr/bin/unshare && mkfs="unshare --user --map-root-user $mkfs" else - print "ERROR: populating filesystem is only supported with ext file systems" + print "ERROR: populating the filesystem is only supported with ext file systems" cleanup_and_exit 1 fi fi @@ -478,12 +479,12 @@ echo "Creating $fs filesystem on $img" export MKE2FS_SYNC=0 - if ! $mkfs $labelopt $mkfs_extra_options "$img" ; then + if ! $mkfs $labelopt $mkfs_extra_options $mkfs_populate_options "$img" ; then if test -z "$mkfs_extra_options" ; then cleanup_and_exit 3 else echo "Filesystem creation failed, trying again without extra options..." - $mkfs $labelopt "$img" || cleanup_and_exit 4 + $mkfs $labelopt $mkfs_populate_options "$img" || cleanup_and_exit 4 fi fi if test -n "$tunefs" ; then @@ -1036,6 +1037,7 @@ echo "RUN_SHELL='$RUN_SHELL'" >> $BUILD_ROOT/.build/build.data echo "RUN_SHELL_AFTER_FAIL='$RUN_SHELL_AFTER_FAIL'" >> $BUILD_ROOT/.build/build.data echo "RUN_SHELL_CMD='$RUN_SHELL_CMD'" >> $BUILD_ROOT/.build/build.data + echo "BUILD_TOOL_OPTS='$BUILD_TOOL_OPTS'" >> $BUILD_ROOT/.build/build.data echo "DO_STATISTICS='$DO_STATISTICS'" >> $BUILD_ROOT/.build/build.data echo "TIME_PREINSTALL='$TIME_PREINSTALL'" >> $BUILD_ROOT/.build/build.data echo "VM_WATCHDOG='$VM_WATCHDOG'" >> $BUILD_ROOT/.build/build.data diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/build-vm-docker new/obs-build-20230808/build-vm-docker --- old/obs-build-20230628/build-vm-docker 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/build-vm-docker 2023-08-08 17:34:27.000000000 +0200 @@ -51,7 +51,6 @@ vm_fixup_docker() { # we need this as marker for handling container specific code. Docker is setting it as well touch $BUILD_ROOT/.dockerenv - test -n "$VM_TYPE_PRIVILEGED" && touch $BUILD_ROOT/.dockerenv.privileged } vm_attach_root_docker() { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/build-vm-kvm new/obs-build-20230808/build-vm-kvm --- old/obs-build-20230628/build-vm-kvm 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/build-vm-kvm 2023-08-08 17:34:27.000000000 +0200 @@ -177,6 +177,19 @@ # from qemu-microvm, minified and hardened PC bios (slightly faster than seabios) test -e /usr/share/qemu/qboot.rom && kvm_options="$kvm_options -bios /usr/share/qemu/qboot.rom" ;; + riscv64) + kvm_bin="/usr/bin/qemu-system-riscv64" + kvm_options="-M virt,accel=kvm" + kvm_cpu="-cpu rv64,sv48=on" + vm_kernel=/boot/Image + vm_initrd=/boot/initrd + test -e /boot/kernel.obs.guest && vm_kernel=/boot/kernel.obs.guest + test -e /boot/initrd.obs.guest && vm_initrd=/boot/initrd.obs.guest + kvm_device=virtio-blk-device + kvm_serial_device=virtio-serial-device,max_ports=2 + kvm_console=hvc0 + kvm_rng_device=virtio-rng-device + ;; esac # check if we can run kvm diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/build-vm-podman new/obs-build-20230808/build-vm-podman --- old/obs-build-20230628/build-vm-podman 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/build-vm-podman 2023-08-08 17:34:27.000000000 +0200 @@ -47,15 +47,10 @@ } vm_fixup_podman() { - # create a scratch image for our build environment if ! podman image exists build-scratch:latest ; then echo "FROM scratch" | podman build -t build-scratch:latest - fi - - # we need this as marker for handling container specific code. Docker is setting it as well - touch $BUILD_ROOT/.podmanenv - test -n "$VM_TYPE_PRIVILEGED" && touch $BUILD_ROOT/.podmanenv.privileged } vm_attach_root_podman() { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/dist/build.changes new/obs-build-20230808/dist/build.changes --- old/obs-build-20230628/dist/build.changes 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/dist/build.changes 2023-08-08 17:34:27.000000000 +0200 @@ -1,4 +1,17 @@ ------------------------------------------------------------------- +Tue Aug 8 15:30:05 UTC 2023 - Adrian Schröter <adr...@suse.de> + +- pbuild: + * fix dependency expansion for build types other than spec + * Reworked cycle handling code + * add --extra-packs option + * add debugflags option +- Pass-through --buildtool-opt +- Parse Patch and Source lines more accurately +- fix tunefs functionality +- minor bugfixes + +------------------------------------------------------------------- Wed Jun 28 07:50:46 UTC 2023 - Adrian Schröter <adr...@suse.de> - --vm-type=podman added (supports also root-less builds) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/initvm.c new/obs-build-20230808/initvm.c --- old/obs-build-20230628/initvm.c 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/initvm.c 2023-08-08 17:34:27.000000000 +0200 @@ -20,7 +20,7 @@ * * AUTHOR * Copyright (c) 2012 James Perkins <james.perk...@linuxfoundation.org> - * Adrian Schroeter <adr...@suse.de> + * Adrian Schroeter <adr...@suse.de> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 or 3 as @@ -49,7 +49,7 @@ /* to enable debugging, compile with -DDEBUG */ #ifdef DEBUG -#define DBG(x) do { x; } while(0) +#define DBG(x) do { x; } while(0) #else #define DBG(x) #endif @@ -330,9 +330,9 @@ int retval; char buf[BUFSIZ], *build_dir; - /* Docker builds get /proc mounted */ + /* Docker builds get /proc mounted, so test for /proc/self */ struct stat sb; - if (stat("/.dockerenv", &sb)) { + if (lstat("/proc/self", &sb)) { /* mount proc filesystem if it isn't already. */ if (mount("proc", "/proc", "proc", MS_MGC_VAL, NULL) == -1) { if (errno != EBUSY) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/obs-build-20230628/pbuild new/obs-build-20230808/pbuild --- old/obs-build-20230628/pbuild 2023-06-28 09:52:29.000000000 +0200 +++ new/obs-build-20230808/pbuild 2023-08-08 17:34:27.000000000 +0200 @@ -104,6 +104,7 @@ # tweak options die("Option --shell only works with --single\n") if $opts->{'shell'} && !$opts->{'single'}; die("Option --shell-after-fail only works with --single\n") if $opts->{'shell-after-fail'} && !$opts->{'single'}; +die("Option --extra-packs only works with --single\n") if $opts->{'extra-packs'} && !$opts->{'single'}; $opts->{'showlog'} = 1 if $opts->{'shell'} || $opts->{'shell-after-fail'} || $opts->{'single'}; $opts->{'buildjobs'} = 1 if $opts->{'showlog'} || $opts->{'single'}; @@ -200,7 +201,7 @@ print("Preset $preset->{'name'} does not contain a build setup definition.\n"); print("Please specify a distribution with the --dist option.\n"); exit; - } + } my @presetnames = PBuild::Preset::known_presets($dir); if (@presetnames) { print("Please specify a distribution or a preset!\n\n"); @@ -502,6 +503,23 @@ # check status of all packages my $result = $ctx->pkgcheck(\@builders, @pkgs); + # finish if extra-packs was used as we cannot use the result + if ($opts->{'extra-packs'}) { + my $pkg = $opts->{'single'}; + die unless $pkg; + my $code = $result->{$pkg}->{'code'}; + my $details = ''; + if ($code eq 'building') { + my @building = map {$_->{'job'}} grep {$_->{'job'}} @builders; + my $job = PBuild::Job::waitjob($opts, @building); + ($code) = PBuild::Job::finishjob($job); + } else { + $details = ": $result->{$pkg}->{'details'}" if $result->{$pkg}->{'details'}; + } + print "$pkg: $code$details\n"; + exit($code eq 'broken' || $code eq 'failed' || $code eq 'unresolvable' ? 1 : 0); + } + # mix in old result from other packages if in single package mode if ($opts->{'single'}) { my $pkg = $opts->{'single'};