After apply the fix and build, I found call function
${@oe.spdx_common.create_spdx_source_deps(d)} or
${oe.spdx_common.create_spdx_source_deps(d)} along with addtask not working
+addtask do_create_spdx after \
+ do_package do_packagedata \
+ do_collect_spdx_deps \
+ ${@oe.spdx_common.create_spdx_source_deps(d)} \
+ before do_populate_sdk do_build do_rm_work
I will try to find a fix in v2
//Hongxu
________________________________
From: [email protected]
<[email protected]> on behalf of hongxu
<[email protected]>
Sent: Tuesday, October 29, 2024 10:59 AM
To: Joshua Watt <[email protected]>
Cc: [email protected]
<[email protected]>
Subject: Re: [oe-core][PATCH 5/6] create-spdx-{2.2,3.0}: support SPDX include
source for work-share directory
Base on your comments, I found we should use create_spdx_source_deps to maintain
the dependency of do_create_spdx for SPDX_INCLUDE_SOURCES = "1"
Move create_spdx_source_deps to
meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
for both
of SPDX 2.2 and SPDX 3.0, use ${@oe.spdx_common.create_spdx_source_deps(d)}
to instead of ${create_spdx_source_deps(d)}. Assure the patched source
in work share is ready for SPDX source copy, like following:
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -588,8 +588,11 @@ python do_create_spdx() {
oe.sbom.write_doc(d, package_doc, pkg_arch, "packages",
indent=get_json_indent(d))
}
do_create_spdx[vardepsexclude] += "BB_NUMBER_THREADS"
-# NOTE: depending on do_unpack is a hack that is necessary to get it's
dependencies for archive the source
-addtask do_create_spdx after do_package do_packagedata do_unpack
do_collect_spdx_deps before do_populate_sdk do_build do_rm_work
+addtask do_create_spdx after \
+ do_package do_packagedata \
+ do_collect_spdx_deps \
+ ${@oe.spdx_common.create_spdx_source_deps(d)} \
+ before do_populate_sdk do_build do_rm_work
SSTATETASKS += "do_create_spdx"
do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
diff --git a/meta/classes/create-spdx-3.0.bbclass
b/meta/classes/create-spdx-3.0.bbclass
index 5f0590198f..d615edcf57 100644
--- a/meta/classes/create-spdx-3.0.bbclass
+++ b/meta/classes/create-spdx-3.0.bbclass
@@ -132,22 +132,9 @@ addtask do_create_spdx after \
do_collect_spdx_deps \
do_deploy_source_date_epoch \
do_populate_sysroot do_package do_packagedata \
- ${create_spdx_source_deps(d)} \
+ ${@oe.spdx_common.create_spdx_source_deps(d)} \
before do_populate_sdk do_populate_sdk_ext do_build do_rm_work
-def create_spdx_source_deps(d):
- deps = []
- if d.getVar("SPDX_INCLUDE_SOURCES") == "1":
- deps.extend([
- # do_unpack is a hack for now; we only need it to get the
- # dependencies do_unpack already has so we can extract the source
- # ourselves
- "do_unpack",
- # For kernel source code
- "do_shared_workdir",
- ])
- return " ".join(deps)
---
a/meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
+++
b/meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
@@ -37,8 +37,35 @@ def extract_licenses(filename):
return []
+def create_spdx_source_deps(d):
+ deps = []
+ if d.getVar("SPDX_INCLUDE_SOURCES") == "1":
+ deps.extend([
+ # do_unpack is a hack for now; we only need it to get the
+ # dependencies do_unpack already has so we can extract the source
+ # ourselves
+ "do_unpack",
+ ])
+
+ # Assure the patched source in work share is ready for SPDX source copy
+ if is_work_shared_spdx(d) and process_sources(d):
+ deps.extend([
+ # For kernel source code
+ "do_shared_workdir",
+ # For gcc-source-${PV}
+ "do_preconfigure",
+ # For gcc-cross-x86_64
+ "do_configure",
+ "do_patch",
+ ])
+
+
+ return " ".join(deps)
//Hongxu
On Fri, Oct 25, 2024 at 9:25 PM Hongxu Jia <[email protected]> wrote:
>
> Originally, while SPDX_INCLUDE_SOURCES = "1" [1], there is bug in scan
> for gcc, libgcc in which the sources locates in work-share directory.
> Copy source from ${WORKDIR} to ${SPDXWORK} did not satisfy the situation
> while ${S} was not included in ${WORKDIR}
>
> This commit aim to support SPDX include source for work-share directory
>
> 1. If is_work_shared_spdx, Copy source from ${S} to ${SPDXWORK},
> normally the dest dir in ${SPDXWORK} has the same basename dir of ${S};
> but for kernel source, rename basename dir 'kernel-source' to ${BP}
> (${BPN}-${PV})
>
> 2. For SPDX source copy, do hard link copy to save copy time
>
> 3. Move do_patch to no work shared situation along with do_unpack
>
> 4. Tweak task do_create_spdx dependencies to assure the patched source
> in work share is ready for SPDX source copy
>
> 5. Remove bb.data.inherits_class('kernel', d) from is_work_shared_spdx,
> the kernel source locates in 'work-shared', test kernel.bbclass is not
> necessary
>
> [1]
> https://docs.yoctoproject.org/dev/ref-manual/variables.html#term-SPDX_INCLUDE_SOURCES<https://urldefense.com/v3/__https://docs.yoctoproject.org/dev/ref-manual/variables.html*term-SPDX_INCLUDE_SOURCES__;Iw!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMUft5bJ9A$>
>
> Signed-off-by: Hongxu Jia <[email protected]>
> ---
> meta/classes/create-spdx-2.2.bbclass | 20 ++++++++++++++
> meta/classes/create-spdx-3.0.bbclass | 19 +++++++++++++
>
> meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
> | 41 +++++++++++-----------------
> 3 files changed, 55 insertions(+), 25 deletions(-)
>
> diff --git a/meta/classes/create-spdx-2.2.bbclass
> b/meta/classes/create-spdx-2.2.bbclass
> index cd1d6819bf..900595a7e9 100644
> --- a/meta/classes/create-spdx-2.2.bbclass
> +++ b/meta/classes/create-spdx-2.2.bbclass
> @@ -947,3 +947,23 @@ def combine_spdx(d, rootfs_name, rootfs_deploydir,
> rootfs_spdxid, packages, spdx
> tar.addfile(info, fileobj=index_str)
>
> combine_spdx[vardepsexclude] += "BB_NUMBER_THREADS
> SPDX_MULTILIB_SSTATE_ARCHS"
> +
> +python () {
> + import oe.spdx_common
> +
> + # Assure the patched source in work share is ready for SPDX source copy
> + if oe.spdx_common.is_work_shared_spdx(d) and \
> + d.getVar("SPDX_INCLUDE_SOURCES") == "1" and \
> + oe.spdx_common.process_sources(d):
> + pn = d.getVar('PN')
> + # There is a corner case with "gcc-source-${PV}" recipes, they use
> "do_preconfigure"
> + # to modify source file after do_patch
> + if oe.spdx_common.has_task(d, "do_preconfigure"):
> + d.appendVarFlag('do_create_spdx', 'depends', '
> %s:do_preconfigure' % pn)
> + elif oe.spdx_common.has_task(d, "do_patch"):
> + d.appendVarFlag('do_create_spdx', 'depends', ' %s:do_patch' % pn)
> + # There is a corner case with "gcc-cross-x86_64" recipes, it has
> "do_configure"
> + # but no do_patch
> + elif oe.spdx_common.has_task(d, "do_configure"):
> + d.appendVarFlag('do_create_spdx', 'depends', ' %s:do_configure'
> % pn)
> +}
For some reason, I think I'd prefer something like:
do_create_spdx[depends] = "${@extra_create_spdx_deps(d)}"
def extra_create_spdx_deps(d):
if BLAH:
return "do_configure"
# ETC.
return ""
But I don't know why. Perhaps Richard can weigh in on if one is better
than the other?
> diff --git a/meta/classes/create-spdx-3.0.bbclass
> b/meta/classes/create-spdx-3.0.bbclass
> index 5f0590198f..284e0ff91c 100644
> --- a/meta/classes/create-spdx-3.0.bbclass
> +++ b/meta/classes/create-spdx-3.0.bbclass
> @@ -190,3 +190,22 @@ python spdx30_build_started_handler () {
> addhandler spdx30_build_started_handler
> spdx30_build_started_handler[eventmask] = "bb.event.BuildStarted"
>
> +python () {
> + import oe.spdx_common
> +
> + # Assure the patched source in work share is ready for SPDX source copy
> + if oe.spdx_common.is_work_shared_spdx(d) and \
> + d.getVar("SPDX_INCLUDE_SOURCES") == "1" and \
> + oe.spdx_common.process_sources(d):
> + pn = d.getVar('PN')
> + # There is a corner case with "gcc-source-${PV}" recipes, they use
> "do_preconfigure"
> + # to modify source file after do_patch
> + if oe.spdx_common.has_task(d, "do_preconfigure"):
> + d.appendVarFlag('do_create_spdx', 'depends', '
> %s:do_preconfigure' % pn)
> + elif oe.spdx_common.has_task(d, "do_patch"):
> + d.appendVarFlag('do_create_spdx', 'depends', ' %s:do_patch' % pn)
> + # There is a corner case with "gcc-cross-x86_64" recipes, it has
> "do_configure"
> + # but no do_patch
> + elif oe.spdx_common.has_task(d, "do_configure"):
> + d.appendVarFlag('do_create_spdx', 'depends', ' %s:do_configure'
> % pn)
> +}
> diff --git
> a/meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
>
> b/meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
> index 1ea55419ae..ab037b4088 100644
> ---
> a/meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
> +++
> b/meta/lib/oe/spdx_common.py<https://urldefense.com/v3/__http://spdx_common.py__;!!AjveYdw8EvQ!bmiw_miqypZcRLzkLoy9hfrLIwIcLFUR6PxuI9Xg7nmkvLhcGSKs8Gw9qp1emglEDbHWz5tsS9uqFcgYse8cIX_5MMVjV0bu4w$>
> @@ -38,7 +38,7 @@ def extract_licenses(filename):
>
>
> def is_work_shared_spdx(d):
> - return bb.data.inherits_class("kernel", d) or ("work-shared" in
> d.getVar("WORKDIR"))
> + return 'work-shared' in d.getVar('S')
Should the "/" be included? (e.g. return "/work-shared/" in d.getVar("S") )
>
>
> def load_spdx_license_data(d):
> @@ -74,11 +74,6 @@ def process_sources(d):
> if d.getVar("PN") == "shadow-sysroot":
> return False
>
> - # We just archive gcc-source for all the gcc related recipes
> - if d.getVar("BPN") in ["gcc", "libgcc"]:
> - bb.debug(1, "spdx: There is bug in scan of %s is, do nothing" % pn)
> - return False
> -
> return True
>
>
> @@ -190,34 +185,28 @@ def get_patched_src(d):
> bb.utils.mkdirhier(d.getVar("B"))
>
> bb.build.exec_func("do_unpack", d)
> - # Copy source of kernel to spdx_workdir
> +
> + if d.getVar("SRC_URI") != "":
> + bb.build.exec_func("do_patch", d)
> +
> + # Copy source from work-share to spdx_workdir
> if is_work_shared_spdx(d):
> - share_src = d.getVar("WORKDIR")
> + share_src = d.getVar('S')
> d.setVar("WORKDIR", spdx_workdir)
> d.setVar("STAGING_DIR_NATIVE", spdx_sysroot_native)
> + # Copy source to ${SPDXWORK}, same basename dir of ${S};
> src_dir = (
> spdx_workdir
> + "/"
> - + d.getVar("PN")
> - + "-"
> - + d.getVar("PV")
> - + "-"
> - + d.getVar("PR")
> + + os.path.basename(share_src)
> )
> - bb.utils.mkdirhier(src_dir)
> + # For kernel souce, rename suffix dir 'kernel-source'
> + # to ${BP} (${BPN}-${PV})
> if bb.data.inherits_class("kernel", d):
> - share_src = d.getVar("STAGING_KERNEL_DIR")
> - cmd_copy_share = "cp -rf " + share_src + "/* " + src_dir + "/"
> - cmd_copy_shared_res = os.popen(cmd_copy_share).read()
> - bb.note("cmd_copy_shared_result = " + cmd_copy_shared_res)
> -
> - git_path = src_dir + "/.git"
> - if os.path.exists(git_path):
> - shutils.rmtree(git_path)
> + src_dir = spdx_workdir + "/" + d.getVar('BP')
>
> - # Make sure gcc and kernel sources are patched only once
> - if not (d.getVar("SRC_URI") == "" or is_work_shared_spdx(d)):
> - bb.build.exec_func("do_patch", d)
> + bb.note(f"copyhardlinktree {share_src} to {src_dir}")
> + oe.path.copyhardlinktree(share_src, src_dir)
>
> # Some userland has no source.
> if not os.path.exists(spdx_workdir):
> @@ -225,6 +214,8 @@ def get_patched_src(d):
> finally:
> d.setVar("WORKDIR", workdir)
>
> +def has_task(d, task):
> + return bool(d.getVarFlag(task, "task", False)) and not
> bool(d.getVarFlag(task, "noexec", False))
>
> def fetch_data_to_uri(fd, name):
> """
> --
> 2.25.1
>
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#206474):
https://lists.openembedded.org/g/openembedded-core/message/206474
Mute This Topic: https://lists.openembedded.org/mt/109220135/21656
Group Owner: [email protected]
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[[email protected]]
-=-=-=-=-=-=-=-=-=-=-=-
