On Tue, 2026-06-16 at 10:25 +0200, Anders Heimer wrote: > Convert the copydebugsources command pipelines to explicit Popen calls > using argument lists. Use env= for LC_ALL, cwd= for the cpio working > directory and glob.glob() for the externalsrc move. > > The first pipeline keeps ignoring command failures as before since some > inputs are expected to fail. The symlink fixup pipeline checks each stage > so failures are reported directly. > > Skip the externalsrc mv when the glob has no matches and let the > following empty-directory cleanup handle the empty tree. > > Signed-off-by: Anders Heimer <[email protected]> > --- > meta/lib/oe/package.py | 63 ++++++++++++++++++++++++++++++------------ > 1 file changed, 45 insertions(+), 18 deletions(-) > > diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py > index c375acc124..ad4b7a2769 100644 > --- a/meta/lib/oe/package.py > +++ b/meta/lib/oe/package.py > @@ -1017,26 +1017,50 @@ def copydebugsources(debugsrcdir, sources, d): > bb.utils.mkdirhier(basepath) > cpath.updatecache(basepath) > > - for pmap in prefixmap: > + env = os.environ.copy() > + env["LC_ALL"] = "C" > + > + for pmap, prefix in prefixmap.items(): > + dstroot = dvar + prefix > # Ignore files from the recipe sysroots (target and native) > - cmd = "LC_ALL=C ; sort -z -u '%s' | egrep -v -z > '((<internal>|<built-in>)$|/.*recipe-sysroot.*/)' | " % sourcefile > + sort_p = subprocess.Popen(["sort", "-z", "-u", "--", > sourcefile], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) > + egrep_p = subprocess.Popen(["egrep", "-v", "-z", "-e", > r"((<internal>|<built-in>)$|/.*recipe-sysroot.*/)"], stdin=sort_p.stdout, > stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env) > + sort_p.stdout.close() > + > # We need to ignore files that are not actually ours > # we do this by only paying attention to items from this package > - cmd += "fgrep -zw '%s' | " % prefixmap[pmap] > + fgrep_p = subprocess.Popen(["fgrep", "-zw", "-e", prefix], > stdin=egrep_p.stdout, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, > env=env) > + egrep_p.stdout.close() > + > # Remove prefix in the source paths > - cmd += "sed 's#%s/##g' | " % (prefixmap[pmap]) > - cmd += "(cd '%s' ; cpio -pd0mlLu --no-preserve-owner '%s%s' > 2>/dev/null)" % (pmap, dvar, prefixmap[pmap]) > + sed_p = subprocess.Popen(["sed", "s#%s/##g" % prefix], > stdin=fgrep_p.stdout, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, > env=env) > + fgrep_p.stdout.close() > + > + cpio_p = subprocess.Popen(["cpio", "-pd0mlLu", > "--no-preserve-owner", dstroot], stdin=sed_p.stdout, cwd=pmap, > stderr=subprocess.DEVNULL, env=env) > + sed_p.stdout.close() > + > + for proc in (cpio_p, sed_p, fgrep_p, egrep_p, sort_p): > + proc.wait()
Hi Anders, thanks for the patches! If we're reworking this code, I think we should replace the complex sed/grep/sort pipeline with Python code. We can read into a Python list and sort/filter using the Python standard library, then pass the results to cpio. > > - try: > - subprocess.check_output(cmd, shell=True, > stderr=subprocess.STDOUT) > - except subprocess.CalledProcessError: > - # Can "fail" if internal headers/transient sources are > attempted > - pass > # cpio seems to have a bug with -lL together and symbolic links > are just copied, not dereferenced. > # Work around this by manually finding and copying any symbolic > links that made it through. > - cmd = "find %s%s -type l -print0 -delete | sed s#%s%s/##g | (cd > '%s' ; cpio -pd0mL --no-preserve-owner '%s%s')" % \ > - (dvar, prefixmap[pmap], dvar, prefixmap[pmap], pmap, > dvar, prefixmap[pmap]) > - subprocess.check_output(cmd, shell=True, > stderr=subprocess.STDOUT) > + # The source copy pipeline above can fail without aborting, so > there may be no copied tree to scan for symlinks. > + if not os.path.exists(dstroot): > + continue > + > + find_p = subprocess.Popen(["find", dstroot, "-type", "l", > "-print0", "-delete"], stdout=subprocess.PIPE) > + sed_p = subprocess.Popen(["sed", "s#%s/##g" % dstroot], > stdin=find_p.stdout, stdout=subprocess.PIPE) > + find_p.stdout.close() > + > + cpio_p = subprocess.Popen(["cpio", "-pd0mL", > "--no-preserve-owner", dstroot], stdin=sed_p.stdout, > stderr=subprocess.DEVNULL, cwd=pmap) > + sed_p.stdout.close() > + > + procs = (cpio_p, sed_p, find_p) > + for proc in procs: > + proc.wait() > + for proc in procs: > + if proc.returncode: > + raise subprocess.CalledProcessError(proc.returncode, > proc.args) This is a simpler pipeline but it may still be nicer to bring it into Python code. > > # debugsources.list may be polluted from the host if we used > externalsrc, > # cpio uses copy-pass and may have just created a directory structure > @@ -1046,13 +1070,16 @@ def copydebugsources(debugsrcdir, sources, d): > > # Same check as above for externalsrc > if workdir not in sdir: > - if os.path.exists(dvar + debugsrcdir + sdir): > - cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir, > dvar,debugsrcdir) > - subprocess.check_output(cmd, shell=True, > stderr=subprocess.STDOUT) > + srcdir = dvar + debugsrcdir + sdir > + dstdir = dvar + debugsrcdir > + if os.path.exists(srcdir): > + entries = glob.glob(os.path.join(glob.escape(srcdir), "*")) > + if entries: > + subprocess.check_output(["mv", "--"] + entries + > [dstdir], stderr=subprocess.STDOUT) We could simply use shutil.move(). > > # The copy by cpio may have resulted in some empty directories! > Remove these > - cmd = "find %s%s -empty -type d -delete" % (dvar, debugsrcdir) > - subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) > + cmd = ["find", dvar + debugsrcdir, "-empty", "-type", "d", "-delete"] > + subprocess.check_output(cmd, stderr=subprocess.STDOUT) I think this is would be more complex in Python code so it is probably best left as a find command. Best regards, -- Paul Barker
signature.asc
Description: This is a digitally signed message part
-=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#238894): https://lists.openembedded.org/g/openembedded-core/message/238894 Mute This Topic: https://lists.openembedded.org/mt/119830169/21656 Group Owner: [email protected] Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [[email protected]] -=-=-=-=-=-=-=-=-=-=-=-
