From: Benjamin Robin <[email protected]>
The source files saved in `${PN}-debugsources.json.zstd` are now saved as
is: "/usr/src/kernel/" is no longer replaced by "${BP}/". However, the
file list is still filtered to exclude "<internal>" or "<built-in>", ...
With this modification, the content of `${PN}-debugsources.json.zstd`
is no longer specific or tied to SPDX generation.
The logic that modified the source file paths to match what SPDX
generation expects is now in `oe.spdx_common.get_compiled_sources()`.
Furthermore, previously, for all non-kernel recipes the source paths were
never "resolved", since the KERNEL_SRC_PATH variable is always defined.
So the returned source file paths always started with
`/usr/src/debug/${PN}/${PV}` (the value of TARGET_DBGSRC_DIR).
Update the source file paths to match what the SPDX generation expects:
- In `get_patched_src()`, the sources of the recipe are extracted
(again). The unpack task runs in a modified local context with
UNPACKDIR set to the value of `${SPDXWORK}`, so the sources end up
extracted in a sub-directory of `${SPDXWORK}`.
- In `add_package_files()`, all files below topdir, set to `${SPDXWORK}`,
are (recursively) listed. For each source file, if its path (relative
to topdir) is in the list returned by `get_compiled_sources()`, the
file is added to the SPDX SBoM.
Handle this by replacing ${TARGET_DBGSRC_DIR} with the relative path of
${S} relative to ${UNPACKDIR}. If ${S} is not relative to ${UNPACKDIR},
do nothing.
Signed-off-by: Benjamin Robin <[email protected]>
Signed-off-by: Mathieu Dubois-Briand <[email protected]>
Signed-off-by: Richard Purdie <[email protected]>
(cherry picked from commit b567c2f0d91fbf36886d0c3a6f6ea90c07771044)
Signed-off-by: Kazuyoshi Akiyama <[email protected]>
---
meta/lib/oe/package.py | 54 +++++++++++++++++++++-------------------------
meta/lib/oe/spdx_common.py | 42 ++++++++++++++++++++++++++++++------
2 files changed, 61 insertions(+), 35 deletions(-)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index c375acc124..bef3c92a25 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -1060,37 +1060,33 @@ def copydebugsources(debugsrcdir, sources, d):
os.rmdir(p)
@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
-def save_debugsources_info(debugsrcdir, sources_raw, d):
+def save_debugsources_info(sources_raw, d):
import json
import bb.compress.zstd
- if debugsrcdir and sources_raw:
- debugsources_file =
d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
- debugsources_dir = os.path.dirname(debugsources_file)
- if not os.path.isdir(debugsources_dir):
- bb.utils.mkdirhier(debugsources_dir)
- bb.utils.remove(debugsources_file)
- workdir = d.getVar("WORKDIR")
- pn = d.getVar('PN')
-
- # Kernel sources are in a different directory and are special case
- # we format the sources as expected by spdx by replacing
/usr/src/kernel/
- # into BP/
- kernel_src = d.getVar('KERNEL_SRC_PATH')
- bp = d.getVar('BP')
- sources_dict = {}
- for file, src_files in sources_raw:
- file_clean = file.replace(f"{workdir}/package/","")
- sources_clean = [
- src.replace(f"{debugsrcdir}/{pn}/", "")
- if not kernel_src else src.replace(f"{kernel_src}/", f"{bp}/")
- for src in src_files
- if not any(keyword in src for keyword in ("<internal>",
"<built-in>")) and not src.endswith("/")
- ]
- sources_dict[file_clean] = sorted(sources_clean)
- num_threads = int(d.getVar("BB_NUMBER_THREADS"))
- with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8",
num_threads=num_threads) as f:
- json.dump(sources_dict, f, sort_keys=True)
+ if not sources_raw:
+ return
+
+ debugsources_file =
d.expand("${PKGDESTWORK}/debugsources/${PN}-debugsources.json.zstd")
+ debugsources_dir = os.path.dirname(debugsources_file)
+ if not os.path.isdir(debugsources_dir):
+ bb.utils.mkdirhier(debugsources_dir)
+ bb.utils.remove(debugsources_file)
+
+ workdir = d.getVar("WORKDIR")
+
+ def _filter_src_file(src):
+ if src.endswith("/"):
+ return False
+ return not any(keyword in src for keyword in ("<internal>",
"<built-in>"))
+
+ sources_dict = {}
+ for file, src_files in sources_raw:
+ file_clean = file.replace(f"{workdir}/package/", "")
+ sources_dict[file_clean] = sorted(filter(_filter_src_file, src_files))
+ num_threads = int(d.getVar("BB_NUMBER_THREADS"))
+ with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8",
num_threads=num_threads) as f:
+ json.dump(sources_dict, f, sort_keys=True)
@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
def read_debugsources_info(d):
@@ -1337,7 +1333,7 @@ def process_split_and_strip_files(d):
copydebugsources(dv["srcdir"], sources, d)
# Save source info to be accessible to other tasks
- save_debugsources_info(dv["srcdir"], results, d)
+ save_debugsources_info(results, d)
#
# End of debug splitting
#
diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py
index e40cf36bba..f885f1832a 100644
--- a/meta/lib/oe/spdx_common.py
+++ b/meta/lib/oe/spdx_common.py
@@ -277,14 +277,44 @@ def get_compiled_sources(d):
bb.debug(1, "Do not have debugsources.list. Skipping")
return [], []
- # Sources are not split now in SPDX, so we aggregate them
- sources = set(itertools.chain.from_iterable(source_info.values()))
- # Check extensions of files
+ unpackdir = d.getVar("UNPACKDIR")
+ srcdir = d.getVar("S")
+ bp = d.getVar("BP")
+ kernel_src = d.getVar("KERNEL_SRC_PATH")
+ dbgsrc_dir = d.getVar("TARGET_DBGSRC_DIR")
+
+ # Compute the relative path of source directory from ${UNPACKDIR}.
+ # The goal is to replace ${TARGET_DBGSRC_DIR} by this relative path.
+ srcdir_rel = None
+ if srcdir and unpackdir:
+ srcdir_rel = os.path.relpath(srcdir, unpackdir)
+ if srcdir_rel.startswith(".."):
+ srcdir_rel = None
+
+ sources = set()
types = set()
- for src in sources:
+
+ # Sources are not split now in SPDX, so we aggregate them
+ for src in set(itertools.chain.from_iterable(source_info.values())):
+ # In the common case, the sources are located in ${S}. To format them
as
+ # expected by SPDX, we replace /usr/src/debug/${PN}/${PV} with the path
+ # of ${S} relative to ${UNPACKDIR}.
+ if dbgsrc_dir and srcdir_rel:
+ src = src.replace(f"{dbgsrc_dir}/", f"{srcdir_rel}/")
+
+ # Kernel sources are in a different directory and are special case
+ # we format the sources as expected by spdx by replacing
/usr/src/kernel/
+ # into ${BP}/
+ if kernel_src and bp:
+ src = src.replace(f"{kernel_src}/", f"{bp}/")
+
+ sources.add(src)
+
+ # Check extensions of files
basename = os.path.basename(src)
ext = basename.partition(".")[2]
- if ext not in types and ext:
+ if ext:
types.add(ext)
- bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)}
{str(types)}")
+
+ bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)}
{types!s}")
return sources, types
--
2.43.0
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#246006):
https://lists.openembedded.org/g/openembedded-core/message/246006
Mute This Topic: https://lists.openembedded.org/mt/121291544/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
