[OE-core] [PATCH 1/1] check-bbclasses: add new script to check bbclasses
FIXES [YOCTO #14235]
This script is a starting point for a "linter" for bbclass files.
Currently it will check for '_' in the bbclass filename and '-' in
addtask or EXPORT_FUNCTION. It will print warnings only no errors.
Signed-off-by: Saul Wold
---
scripts/check-bbclasses | 109
1 file changed, 109 insertions(+)
create mode 100755 scripts/check-bbclasses
diff --git a/scripts/check-bbclasses b/scripts/check-bbclasses
new file mode 100755
index 000..ea525b2d118
--- /dev/null
+++ b/scripts/check-bbclasses
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This script checks for bbclass like a linter can gives a
+# warning if any of the following issues:
+# * bbclass filename contains a '_' unless in the know list
+# * either an 'addtask' or 'EXPORT_FUNCTION' name contains a '-'
+#
+
+import sys, os, subprocess, re, shutil
+
+# List of known classes with '_' in OE-Core
+oecore_known_classes = (
+ "test_events.bbclass",
+ "migrate_localcount.bbclass",
+ "copyleft_compliance.bbclass",
+ "sign_ipk.bbclass",
+ "multilib_global.bbclass",
+ "useradd_base.bbclass",
+ "rm_work_and_downloads.bbclass",
+ "sign_rpm.bbclass",
+ "rm_work.bbclass",
+ "sign_package_feed.bbclass",
+ "copyleft_filter.bbclass",
+ "relative_symlinks.bbclass",
+ "recipe_sanity.bbclass",
+ "metadata_scm.bbclass",
+ "python_pyo3.bbclass",
+ "multilib_script.bbclass",
+ "multilib_header.bbclass",
+ "compress_doc.bbclass",
+ "populate_sdk.bbclass",
+ "license_image.bbclass",
+ "python_maturin.bbclass",
+ "python_setuptools3_rust.bbclass",
+ "image_types.bbclass",
+ "cargo_c.bbclass",
+ "bin_package.bbclass",
+ "python_poetry_core.bbclass",
+ "distro_features_check.bbclass",
+ "lib_package.bbclass",
+ "python_setuptools_build_meta.bbclass",
+ "populate_sdk_base.bbclass",
+ "features_check.bbclass",
+ "python_pep517.bbclass",
+ "cargo_common.bbclass",
+ "populate_sdk_ext.bbclass",
+ "rootfs_ipk.bbclass",
+ "rootfs_deb.bbclass",
+ "cpan_build.bbclass",
+ "rootfs_rpm.bbclass",
+ "python_flit_core.bbclass",
+ "python_hatchling.bbclass",
+ "image_types_wic.bbclass",
+ "setuptools3_legacy.bbclass",
+ "package_rpm.bbclass",
+ "package_deb.bbclass",
+ "package_ipk.bbclass",
+ "package_pkgdata.bbclass"
+)
+
+def get_tinfoil():
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = scripts_path + '/lib'
+sys.path = sys.path + [lib_path]
+import scriptpath
+scriptpath.add_bitbake_lib_path()
+import bb.tinfoil
+tinfoil = bb.tinfoil.Tinfoil()
+tinfoil.prepare()
+# tinfoil.logger.setLevel(logging.WARNING)
+return tinfoil
+
+if __name__=='__main__':
+import argparse, shutil
+
+parser = argparse.ArgumentParser(description='Sanity checker for
bbclasses')
+parser.add_argument("--verbose", default=False, action="store_true")
+args = parser.parse_args()
+
+tinfoil = get_tinfoil()
+
+bbpath = tinfoil.config_data.getVar('BBPATH').split(':')
+for path in bbpath:
+with os.scandir(path) as it:
+for entry in it:
+if "classes" in entry.name and entry.is_dir():
+with os.scandir(path + "/" + entry.name) as classes:
+for c in classes:
+#
+# Check for underscore in bbclass filename
+#
+if c.name.endswith(".bbclass") and "_" in c.name
and not c.name in oecore_known_classes:
+
+print("Warning: BBClass file name contains
'_': " + path + "/" + entry.name + "/" + c.name)
+#
+# Check for '-' in exported functions and tasks
+#
+with open(path + "/" + entry.name + "/" + c.name)
as f:
+for line in f.readlines():
+if line.startswith("addtask ") and "-" in
line:
+
[OE-core] [PATCH 0/1] BBClass checker script (14235)
This is the first pass at a script for addressing 14235 [0]: bbclass file name convention is not consistent wrt dash and underscore This is could be a contrib script or added to bitbake parsing somehow, I am open to suggestions for where to add this or if other tests are needed. [0] https://bugzilla.yoctoproject.org/show_bug.cgi?id=14235 Saul Wold (1): check-bbclasses: add new script to check bbclasses scripts/check-bbclasses | 109 1 file changed, 109 insertions(+) create mode 100755 scripts/check-bbclasses -- 2.34.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#194435): https://lists.openembedded.org/g/openembedded-core/message/194435 Mute This Topic: https://lists.openembedded.org/mt/104005171/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] package.py: OEHasPackage: Add MLPREFIX to packagename
FIXES [YOCTO #12342]
When testing a Multilib image, the package manifest list contains
the fully qualified package name which includes the Multilib Prefix.
This patch adds the MLPREFIX to the package names that are passed
into the @OEHasPackage() decorator to ensure the set isdisjoint()
matches correctly.
Signed-off-by: Saul Wold
---
Tested with a lib32 image and without
meta/lib/oeqa/runtime/decorator/package.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/meta/lib/oeqa/runtime/decorator/package.py
b/meta/lib/oeqa/runtime/decorator/package.py
index 8aba3f325bc..b78ac9fc388 100644
--- a/meta/lib/oeqa/runtime/decorator/package.py
+++ b/meta/lib/oeqa/runtime/decorator/package.py
@@ -38,11 +38,12 @@ class OEHasPackage(OETestDecorator):
if isinstance(self.need_pkgs, str):
self.need_pkgs = [self.need_pkgs,]
+mlprefix = self.case.td.get("MLPREFIX")
for pkg in self.need_pkgs:
if pkg.startswith('!'):
-unneed_pkgs.add(pkg[1:])
+unneed_pkgs.add(mlprefix + pkg[1:])
else:
-need_pkgs.add(pkg)
+need_pkgs.add(mlprefix + pkg)
if unneed_pkgs:
msg = 'Checking if %s is not installed' % ', '.join(unneed_pkgs)
--
2.34.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#192889):
https://lists.openembedded.org/g/openembedded-core/message/192889
Mute This Topic: https://lists.openembedded.org/mt/103342284/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] create-spdx: Add check for variable contents along with quotes
This adds a check to ensure we that if a variable is
empty it gets an empty string, this also adds quotes
to indicate the variable contents.
"comment": "CUSTOM_SPECIAL="
or
"comment": "CUSTOM_SPECIAL=variable contents"
Signed-off-by: Saul Wold
---
meta/classes/create-spdx-2.2.bbclass | 5 ++-
.../selftest/cases/spdx_custom_annotations.py | 42 +++
2 files changed, 46 insertions(+), 1 deletion(-)
create mode 100644 meta/lib/oeqa/selftest/cases/spdx_custom_annotations.py
diff --git a/meta/classes/create-spdx-2.2.bbclass
b/meta/classes/create-spdx-2.2.bbclass
index 454dd7a7a07..da90bf8033a 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -524,7 +524,10 @@ python do_create_spdx() {
if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split():
-recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var)))
+if d.getVar(var):
+recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var).replace('"', '\\"')))
+else:
+recipe.annotations.append(create_annotation(d, var + "="))
# Some CVEs may be patched during the build process without incrementing
the version number,
# so querying for CVEs based on the CPE id can lead to false positives. To
account for this,
diff --git a/meta/lib/oeqa/selftest/cases/spdx_custom_annotations.py
b/meta/lib/oeqa/selftest/cases/spdx_custom_annotations.py
new file mode 100644
index 000..a2bef998988
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/spdx_custom_annotations.py
@@ -0,0 +1,42 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+from oeqa.selftest.case import OESelftestTestCase
+from oeqa.utils.commands import bitbake
+import json
+
+class SPDXCustomAnnotations(OESelftestTestCase):
+
+# Build test recipes with custom-annotation
+def test_spdx_custom_annotations(self):
+self.write_config("INHERIT:append = ' create_spdx'")
+
+result = bitbake('%s custom-annotation')
+print(f"Bitbake Result: {result}")
+try:
+mdir = self.get_dir_from_bb_var('DEPLOY_DIR_SPDX',
self.buildtarget)
+with open(f"{mdir}/recpes/recipe-{pn}.spdx.json") as json_file:
+spdx_json = json.load(json_file)
+print(f'{spdx_json["packages"]["annotations"][0]["comment"]}')
+
self.assertEqual(spdx_json["packages"]["annotations"][0]["comment"],
"ANNOTE1=This is the first custom annotation")
+
self.assertEqual(spdx_json["packages"]["annotations"][1]["comment"],
"ANNOTE2=This is another custom annotation")
+except:
+print("json load failed")
+
+#class SPDXCustomeAnnotationTests(OESelftestTestCase):
+#def default_config(self):
+#return """
+#INHERIT:append = " create-spdx"
+#SPDX_CUSTOM_ANNOTATION_VARS:pn-core-image-minimal = "TEST_VAR"
+#TEST_VAR:pn-core-image-minimal = "This is a test annotation"
+#"""
+#
+#def test_image_annotation(self):
+#self.write_config(self.default_config())
+#
+#result = bitbake('core-image-minimal', ignore_status=True)
+
+
+
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177709):
https://lists.openembedded.org/g/openembedded-core/message/177709
Mute This Topic: https://lists.openembedded.org/mt/97213264/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH][kirkstone] package.bbclase: Add check for /build in copydebugsources()
This is needed when the SDK or eSDK is installed in a /build top level
directory as it conflicts with the build directory within the existing
/usr/src/debug/build (which is really a link). Rename it and then do the
copy, this is not an issue with master currently due to some other
changes that occurred in master.
Fixes: [YOCTO #15026]
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 7 +++
1 file changed, 7 insertions(+)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 8b11fdd155..2950218145 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -636,6 +636,13 @@ def copydebugsources(debugsrcdir, sources, d):
# Same check as above for externalsrc
if workdir not in sdir:
if os.path.exists(dvar + debugsrcdir + sdir):
+# Special case for /build since we need to move into
+# /usr/src/debug/build so rename sdir to build.build
+if sdir.find("/build") == 0:
+cmd = "mv %s%s%s %s%s%s" % (dvar, debugsrcdir, "/build",
dvar, debugsrcdir, "/build.build")
+subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
+sdir = sdir.replace("/build", "/build.build", 1)
+
cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir,
dvar,debugsrcdir)
subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
--
2.35.6
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177480):
https://lists.openembedded.org/g/openembedded-core/message/177480
Mute This Topic: https://lists.openembedded.org/mt/97101388/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] package.bbclase: Add check for /build in copydebugsources()
On 2/20/23 14:40, Richard Purdie wrote:
On Mon, 2023-02-20 at 13:30 -0800, Saul Wold wrote:
This is needed when the SDK or eSDK is installed in a /build top level
directory as it conflicts with the build directory within the existing
/usr/src/debug/build (which is really a link). Rename it and then do the
copy,
this is not an issue with master currently due to some other
changes that occurred in master.
Is this an issue in master or not? I can't parse that! :)
Oops, sorry this is meant for Kirkstone branch, their is a different problem in
master since the code was refactored a direct backport is not possible.
Oops, sorry this is meant for Kirkstone branch, the code in master was
refactored and seems to have solved the problem since /us/src/debug now
has $PN/$PV components in the path.
I will resend it as kirkstone specific.
Sau!
Cheers,
Richard
Fixes: [YOCTO #15026]
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 7 +++
1 file changed, 7 insertions(+)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 8b11fdd155..2950218145 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -636,6 +636,13 @@ def copydebugsources(debugsrcdir, sources, d):
# Same check as above for externalsrc
if workdir not in sdir:
if os.path.exists(dvar + debugsrcdir + sdir):
+# Special case for /build since we need to move into
+# /usr/src/debug/build so rename sdir to build.build
+if sdir.find("/build") == 0:
+cmd = "mv %s%s%s %s%s%s" % (dvar, debugsrcdir, "/build", dvar,
debugsrcdir, "/build.build")
+subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
+sdir = sdir.replace("/build", "/build.build", 1)
+
cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir,
dvar,debugsrcdir)
subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177479):
https://lists.openembedded.org/g/openembedded-core/message/177479
Mute This Topic: https://lists.openembedded.org/mt/97096266/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] package.bbclase: Add check for /build in copydebugsources()
This is needed when the SDK or eSDK is installed in a /build top level
directory as it conflicts with the build directory within the existing
/usr/src/debug/build (which is really a link). Rename it and then do the
copy, this is not an issue with master currently due to some other
changes that occurred in master.
Fixes: [YOCTO #15026]
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 7 +++
1 file changed, 7 insertions(+)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 8b11fdd155..2950218145 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -636,6 +636,13 @@ def copydebugsources(debugsrcdir, sources, d):
# Same check as above for externalsrc
if workdir not in sdir:
if os.path.exists(dvar + debugsrcdir + sdir):
+# Special case for /build since we need to move into
+# /usr/src/debug/build so rename sdir to build.build
+if sdir.find("/build") == 0:
+cmd = "mv %s%s%s %s%s%s" % (dvar, debugsrcdir, "/build",
dvar, debugsrcdir, "/build.build")
+subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
+sdir = sdir.replace("/build", "/build.build", 1)
+
cmd = "mv %s%s%s/* %s%s" % (dvar, debugsrcdir, sdir,
dvar,debugsrcdir)
subprocess.check_output(cmd, shell=True,
stderr=subprocess.STDOUT)
--
2.35.6
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177459):
https://lists.openembedded.org/g/openembedded-core/message/177459
Mute This Topic: https://lists.openembedded.org/mt/97096266/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] create-spdx: Add check for variable contents along with quotes
On 2/16/23 07:25, Joshua Watt wrote:
On Wed, Feb 15, 2023 at 6:31 PM Saul Wold wrote:
This adds a check to ensure we that if a variable is
empty it gets an empty string, this also adds quotes
to indicate the variable contents.
"comment": "CUSTOM_SPECIAL=''"
or
"comment": "CUSTOM_SPECIAL= 'variable contents'"
Signed-off-by: Saul Wold
---
meta/classes/create-spdx-2.2.bbclass | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx-2.2.bbclass
b/meta/classes/create-spdx-2.2.bbclass
index 28a42e009f6..8b18ada40a4 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -484,7 +484,10 @@ python do_create_spdx() {
if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split():
-recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var)))
+if d.getVar(var):
+recipe.annotations.append(create_annotation(d, var + "= '" +
d.getVar(var) + "'"))
Why the extra space after the = ?
Right, probably should remove that.
Does the variable need some escaping in case it already has a single
quote in it?
Was thinking about consistency with the empty case below, but I guess we
need to think not just of a "stray" single quote in the string, but also
a stray double quote (").
I guess we can just remove the single quotes completely for both cases.
Sau!
+else:
+recipe.annotations.append(create_annotation(d, var + "=''"))
# Some CVEs may be patched during the build process without incrementing
the version number,
# so querying for CVEs based on the CPE id can lead to false positives.
To account for this,
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177295):
https://lists.openembedded.org/g/openembedded-core/message/177295
Mute This Topic: https://lists.openembedded.org/mt/96996669/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] create-spdx: Add check for variable contents along with quotes
This adds a check to ensure we that if a variable is
empty it gets an empty string, this also adds quotes
to indicate the variable contents.
"comment": "CUSTOM_SPECIAL=''"
or
"comment": "CUSTOM_SPECIAL= 'variable contents'"
Signed-off-by: Saul Wold
---
meta/classes/create-spdx-2.2.bbclass | 5 -
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx-2.2.bbclass
b/meta/classes/create-spdx-2.2.bbclass
index 28a42e009f6..8b18ada40a4 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -484,7 +484,10 @@ python do_create_spdx() {
if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split():
-recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var)))
+if d.getVar(var):
+recipe.annotations.append(create_annotation(d, var + "= '" +
d.getVar(var) + "'"))
+else:
+recipe.annotations.append(create_annotation(d, var + "=''"))
# Some CVEs may be patched during the build process without incrementing
the version number,
# so querying for CVEs based on the CPE id can lead to false positives. To
account for this,
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177224):
https://lists.openembedded.org/g/openembedded-core/message/177224
Mute This Topic: https://lists.openembedded.org/mt/96996669/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 0/1] Add support for custom annotations in SPDX
V2 fixes commit message and adds additional check for empty
custom_var.
Additional testing with SPDX_CUSTOM_ANNOTATION_VARS:pn-${PN} and
SPDX_CUSTOM_ANNOTATION_VARS:append:pn-${PN} confirmed to work
as expected.
We will leave the variable flag handling for a future extension
of this code as needed.
Saul Wold (1):
create-spdx-2.2: Add support for custom Annotations
meta/classes/create-spdx-2.2.bbclass | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177168):
https://lists.openembedded.org/g/openembedded-core/message/177168
Mute This Topic: https://lists.openembedded.org/mt/96964901/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 1/1] create-spdx-2.2: Add support for custom Annotations
This change adds a new variable to track which recipe variables
are added as SPDX Annotations.
Usage: add SPDX_CUSTOM_ANNOTATION_VARS =
The recipe spdx json will contain an annotation stanza that looks
something like this:
"annotations": [
{
"annotationDate": "2023-02-13T19:44:20Z",
"annotationType": "OTHER",
"annotator": "Tool: oe-spdx-creator - 1.0",
"comment": "CUSTOM_VARIABLE=some value or string"
},
Signed-off-by: Saul Wold
---
meta/classes/create-spdx-2.2.bbclass | 8 +++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx-2.2.bbclass
b/meta/classes/create-spdx-2.2.bbclass
index f0513af083b..bdc2e2c91e7 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -30,6 +30,8 @@ SPDX_PRETTY ??= "0"
SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
+SPDX_CUSTOM_ANNOTATION_VARS ??= ""
+
SPDX_ORG ??= "OpenEmbedded ()"
SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created
from \
@@ -402,7 +404,6 @@ def collect_dep_sources(d, dep_recipes):
return sources
-
python do_create_spdx() {
from datetime import datetime, timezone
import oe.sbom
@@ -479,6 +480,11 @@ python do_create_spdx() {
if description:
recipe.description = description
+if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
+for var in d.getVar("SPDX_CUSTOM_ANNOTATION_VARS").split():
+if d.getVar(var):
+recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var)))
+
# Some CVEs may be patched during the build process without incrementing
the version number,
# so querying for CVEs based on the CPE id can lead to false positives. To
account for this,
# save the CVEs fixed by patches to source information field in the SPDX.
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177167):
https://lists.openembedded.org/g/openembedded-core/message/177167
Mute This Topic: https://lists.openembedded.org/mt/96964900/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] create-spdx-2.2: Add support for custom Annotations
On 2/13/23 12:03, Joshua Watt wrote:
On Mon, Feb 13, 2023 at 1:54 PM Saul Wold wrote:
This change adds a new variable to track which recipe variables
are added as SPDX Annotations.
Usage: add SPDX_CUSTOME_ANNOTATION_VARS =
nit: CUSTOM
v2 will come shortly (I will try to address the flags)
The recipe spdx json will contain an annotation stanza that looks
something like this:
"annotations": [
{
"annotationDate": "2023-02-13T19:44:20Z",
"annotationType": "OTHER",
"annotator": "Tool: oe-spdx-creator - 1.0",
"comment": "CUSTOM_VARIABLE=some value or string"
},
Signed-off-by: Saul Wold
---
meta/classes/create-spdx-2.2.bbclass | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx-2.2.bbclass
b/meta/classes/create-spdx-2.2.bbclass
index f0513af083b..e1bbf646ff9 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -30,6 +30,8 @@ SPDX_PRETTY ??= "0"
SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
+SPDX_CUSTOM_ANNOTATION_VARS ??= ""
+
SPDX_ORG ??= "OpenEmbedded ()"
SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages
created from \
@@ -402,7 +404,6 @@ def collect_dep_sources(d, dep_recipes):
return sources
-
python do_create_spdx() {
from datetime import datetime, timezone
import oe.sbom
@@ -479,6 +480,10 @@ python do_create_spdx() {
if description:
recipe.description = description
+if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
+for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split():
+recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var)))
+
Seems reasonable. If we need more configuration options, I think we
can add it later with flags, e.g.
MY_VAR = "foo"
MY_VAR[spdx-annotator] = "Me!"
SPDX_CUSTOM_ANNOTATION_VARS = "MY_VAR"
What did you think the output should be here? ie what does the comment
line contain?
Today the annotation would contain:
"comment": "MY_VAR=foo"
What should the comment line contain if a flag or multiple flags exists?
Or the CUSTOM_ANNOTATION code only looks for one flag [spdx-annotator]?
"comment": "Me!=foo"
Thoughts?
Aslo, in the future if users want package annotations, we can probably do:
SPDX_CUSTOM_ANNOTATION_VARS:${PN}
Do you really mean SPDX_CUSTOM_ANNOTATIONS_VARS:pn-${PN}
I tested this and it appears to work, along with the :append:pn-${PN} style.
Sau!
# Some CVEs may be patched during the build process without incrementing
the version number,
# so querying for CVEs based on the CPE id can lead to false positives.
To account for this,
# save the CVEs fixed by patches to source information field in the SPDX.
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177160):
https://lists.openembedded.org/g/openembedded-core/message/177160
Mute This Topic: https://lists.openembedded.org/mt/96944341/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] create-spdx-2.2: Add support for custom Annotations
This change adds a new variable to track which recipe variables
are added as SPDX Annotations.
Usage: add SPDX_CUSTOME_ANNOTATION_VARS =
The recipe spdx json will contain an annotation stanza that looks
something like this:
"annotations": [
{
"annotationDate": "2023-02-13T19:44:20Z",
"annotationType": "OTHER",
"annotator": "Tool: oe-spdx-creator - 1.0",
"comment": "CUSTOM_VARIABLE=some value or string"
},
Signed-off-by: Saul Wold
---
meta/classes/create-spdx-2.2.bbclass | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx-2.2.bbclass
b/meta/classes/create-spdx-2.2.bbclass
index f0513af083b..e1bbf646ff9 100644
--- a/meta/classes/create-spdx-2.2.bbclass
+++ b/meta/classes/create-spdx-2.2.bbclass
@@ -30,6 +30,8 @@ SPDX_PRETTY ??= "0"
SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
+SPDX_CUSTOM_ANNOTATION_VARS ??= ""
+
SPDX_ORG ??= "OpenEmbedded ()"
SPDX_SUPPLIER ??= "Organization: ${SPDX_ORG}"
SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for SPDX packages created
from \
@@ -402,7 +404,6 @@ def collect_dep_sources(d, dep_recipes):
return sources
-
python do_create_spdx() {
from datetime import datetime, timezone
import oe.sbom
@@ -479,6 +480,10 @@ python do_create_spdx() {
if description:
recipe.description = description
+if d.getVar("SPDX_CUSTOM_ANNOTATION_VARS"):
+for var in d.getVar('SPDX_CUSTOM_ANNOTATION_VARS').split():
+recipe.annotations.append(create_annotation(d, var + "=" +
d.getVar(var)))
+
# Some CVEs may be patched during the build process without incrementing
the version number,
# so querying for CVEs based on the CPE id can lead to false positives. To
account for this,
# save the CVEs fixed by patches to source information field in the SPDX.
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#177109):
https://lists.openembedded.org/g/openembedded-core/message/177109
Mute This Topic: https://lists.openembedded.org/mt/96944341/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] busybox: Fix depmod patch
The original patch was actually allowing .debug modules though which was in-correct. This change blocks the parsing of .debug modules (which is correct). As noted in [YOCTO #15022] this should address the empty modules.dep when using the BusyBox depmod. Signed-off-by: Saul Wold --- .../busybox/busybox/0001-depmod-Ignore-.debug-directories.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch b/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch index 354f83a4a5f..d76118f85b2 100644 --- a/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch +++ b/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch @@ -21,7 +21,7 @@ index bb42bbe..aa5a2de 100644 /* Arbitrary. Was sb->st_size, but that breaks .gz etc */ size_t len = (64*1024*1024 - 4096); -+ if (strstr(fname, ".debug") == NULL) ++ if (strstr(fname, ".debug") != NULL) + return TRUE; + if (strrstr(fname, ".ko") == NULL) -- 2.39.0 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#176995): https://lists.openembedded.org/g/openembedded-core/message/176995 Mute This Topic: https://lists.openembedded.org/mt/96878297/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] at: Change when files are copied
The create_spdx code relies on patched code, if files are changed
or added during the do_configure phase they will be missed by the
create_spdx process. So we need to ensure files modifications/additions
happen in the do_patch phase.
Signed-off-by: Saul Wold
---
meta/recipes-extended/at/at_3.2.5.bb | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-extended/at/at_3.2.5.bb
b/meta/recipes-extended/at/at_3.2.5.bb
index 87a436173f1..c0c876a6443 100644
--- a/meta/recipes-extended/at/at_3.2.5.bb
+++ b/meta/recipes-extended/at/at_3.2.5.bb
@@ -52,8 +52,10 @@ INITSCRIPT_PARAMS = "defaults"
SYSTEMD_SERVICE:${PN} = "atd.service"
-do_configure:prepend() {
- cp -f ${WORKDIR}/posixtm.[ch] ${S}
+do_patch[postfuncs] += "copy_posix_files"
+
+copy_posix_files() {
+cp -f ${WORKDIR}/posixtm.[ch] ${S}
}
do_install () {
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#175755):
https://lists.openembedded.org/g/openembedded-core/message/175755
Mute This Topic: https://lists.openembedded.org/mt/96203067/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] at: Change where files are modified
The create_spdx code relies on patched code, if files are changed
or added during the do_configure phase they will be missed by the
create_spdx process. So we need to ensure files modifications/additions
happen in the do_patch phase.
Signed-off-by: Saul Wold
---
meta/recipes-extended/at/at_3.2.5.bb | 8 ++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-extended/at/at_3.2.5.bb
b/meta/recipes-extended/at/at_3.2.5.bb
index 87a436173f1..45700366e0f 100644
--- a/meta/recipes-extended/at/at_3.2.5.bb
+++ b/meta/recipes-extended/at/at_3.2.5.bb
@@ -52,8 +52,12 @@ INITSCRIPT_PARAMS = "defaults"
SYSTEMD_SERVICE:${PN} = "atd.service"
-do_configure:prepend() {
- cp -f ${WORKDIR}/posixtm.[ch] ${S}
+do_patch:append() {
+bb.build.exec_func('copy_posix_files', d)
+}
+
+copy_posix_files() {
+cp -f ${WORKDIR}/posixtm.[ch] ${S}
}
do_install () {
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#175753):
https://lists.openembedded.org/g/openembedded-core/message/175753
Mute This Topic: https://lists.openembedded.org/mt/96201863/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] depmodwrapper-cross: Fix missing $
Signed-off-by: Saul Wold
---
meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
index 65068f02df8..303026ad789 100644
--- a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
+++ b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
@@ -30,7 +30,7 @@ if [ -r "${PKGDATA_DIR}/kernel-depmod/kernel-abiversion" ];
then
kernelabi=\$(cat "${PKGDATA_DIR}/kernel-depmod/kernel-abiversion")
fi
-if [ ! -e "\3${nonarch_base_libdir}/depmod.d/exclude.conf" ]; then
+if [ ! -e "\$3${nonarch_base_libdir}/depmod.d/exclude.conf" ]; then
mkdir -p "\$3${nonarch_base_libdir}/depmod.d"
echo "exclude .debug" > "\$3${nonarch_base_libdir}/depmod.d/exclude.conf"
fi
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#164049):
https://lists.openembedded.org/g/openembedded-core/message/164049
Mute This Topic: https://lists.openembedded.org/mt/90270351/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] kmod: Update exclude patch to Accepted
Upstream made a few tweaks and accepted the patch.
Signed-off-by: Saul Wold
---
...dd-support-for-excluding-a-directory.patch | 35 ++-
1 file changed, 19 insertions(+), 16 deletions(-)
diff --git
a/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
index 18d97935331..ea0570af2bf 100644
---
a/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
+++
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
@@ -1,6 +1,6 @@
-From 01f3fe68a7a42b06eb318f3b09fa5e5ea75d46c4 Mon Sep 17 00:00:00 2001
+From f50e2d67575ac5f256fb853ca9d29aeac92d9a57 Mon Sep 17 00:00:00 2001
From: Saul Wold
-Date: Tue, 22 Mar 2022 12:11:45 -0700
+Date: Thu, 31 Mar 2022 14:56:28 -0700
Subject: [PATCH] depmod: Add support for excluding a directory
This adds support to depmod to enable a new exclude directive in
@@ -12,13 +12,15 @@ via a new exclude directive.
depmod.d/exclude.conf example:
exclude.debug
-Upstream-Status: Submitted
+Upstream-Status: Accepted
Signed-off-by: Saul Wold
+[ Fix warnings and make should_exclude_dir() return bool ]
+Signed-off-by: Lucas De Marchi
---
- man/depmod.d.xml | 14 +++
- tools/depmod.c | 65 +---
- 2 files changed, 75 insertions(+), 4 deletions(-)
+ man/depmod.d.xml | 14 ++
+ tools/depmod.c | 66 +---
+ 2 files changed, 76 insertions(+), 4 deletions(-)
diff --git a/man/depmod.d.xml b/man/depmod.d.xml
index b315e93..76548e9 100644
@@ -46,7 +48,7 @@ index b315e93..76548e9 100644
diff --git a/tools/depmod.c b/tools/depmod.c
-index eb810b8..ac365e9 100644
+index 07a35ba..4117dd1 100644
--- a/tools/depmod.c
+++ b/tools/depmod.c
@@ -458,6 +458,11 @@ struct cfg_external {
@@ -125,32 +127,33 @@ index eb810b8..ac365e9 100644
}
-@@ -1229,6 +1270,24 @@ add:
+@@ -1229,6 +1270,25 @@ add:
return 0;
}
-+static int should_exclude_dir(struct cfg *cfg, char *name)
++static bool should_exclude_dir(const struct cfg *cfg, const char *name)
+{
+ struct cfg_exclude *exc;
+
+ if (name[0] == '.' && (name[1] == '\0' ||
+ (name[1] == '.' && name[2] == '\0')))
-+ return 1;
++ return true;
++
+ if (streq(name, "build") || streq(name, "source"))
-+ return 1;
++ return true;
+
+ for (exc = cfg->excludes; exc != NULL; exc = exc->next) {
-+ if (streq(name, exc->exclude_dir)) {
-+ return 1;
-+ }
++ if (streq(name, exc->exclude_dir))
++ return true;
+ }
-+ return 0;
++
++ return false;
+}
+
static int depmod_modules_search_dir(struct depmod *depmod, DIR *d, size_t
baselen, struct scratchbuf *s_path)
{
struct dirent *de;
-@@ -1240,11 +1299,9 @@ static int depmod_modules_search_dir(struct depmod
*depmod, DIR *d, size_t basel
+@@ -1240,11 +1300,9 @@ static int depmod_modules_search_dir(struct depmod
*depmod, DIR *d, size_t basel
size_t namelen;
uint8_t is_dir;
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#163994):
https://lists.openembedded.org/g/openembedded-core/message/163994
Mute This Topic: https://lists.openembedded.org/mt/90242522/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH v2 3/3] depmodwrapper: Use nonarch_base_libdir for depmod.d
On 4/1/22 04:11, Richard Purdie wrote:
On Thu, 2022-03-31 at 15:21 -0700, Saul Wold wrote:
This ensure that when depmod-native runs we can find the correct
exclude.conf information, in this case adding .debug to ignore
the .debug kernell modules. The kmod utilities like depmod can use
either /etc/depmod.d or /lib/depmod.d. The kmod recipe is installing
the existing search.conf to /lib/depmod.d (nonarch_base_lib)
When the busybox modutils are used, /lib/depmod.d is not used, so
it's safe add the exclude.conf file to /lib/depmod.d.
Signed-off-by: Saul Wold
---
meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
index 04fc14a6d21..65068f02df8 100644
--- a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
+++ b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
@@ -30,11 +30,16 @@ if [ -r "${PKGDATA_DIR}/kernel-depmod/kernel-abiversion" ];
then
kernelabi=\$(cat "${PKGDATA_DIR}/kernel-depmod/kernel-abiversion")
fi
+if [ ! -e "\3${nonarch_base_libdir}/depmod.d/exclude.conf" ]; then
+mkdir -p "\$3${nonarch_base_libdir}/depmod.d"
+echo "exclude .debug" > "\$3${nonarch_base_libdir}/depmod.d/exclude.conf"
+fi
Shouldn't the above go into the kmod recipe? We need this on target as well as
in our rootfs build, right? I'm worried about the case where someone calls
depmod on target.
The kmod recipe does install it into nonarch_base_libdir/depmod.d, this
is for the case of busybox which does not install anything and we are
using kmod-native and the code below has depmod-native pointing to the
correct depmod.d in the target rootfs image.
Files installed by kmod-native don't make it to the target rootfs, so we
need to create the exclude.conf here. I could remove it after
depmod-native runs if that's your concern.
+
if [ ! -r ${PKGDATA_DIR}/kernel-depmod/System.map-\$4 ] || [ "\$kernelabi" !=
"\$4" ]; then
echo "Unable to read: ${PKGDATA_DIR}/kernel-depmod/System.map-\$4" >&2
-exec env depmod -C "\$3${sysconfdir}/depmod.d" "\$1" "\$2" "\$3" "\$4"
+exec env depmod -C "\$3${nonarch_base_libdir}/depmod.d" "\$1" "\$2" "\$3"
"\$4"
else
-exec env depmod -C "\$3${sysconfdir}/depmod.d" "\$1" "\$2" "\$3" -F
"${PKGDATA_DIR}/kernel-depmod/System.map-\$4" "\$4"
+exec env depmod -C "\$3${nonarch_base_libdir}/depmod.d" "\$1" "\$2" "\$3" -F
"${PKGDATA_DIR}/kernel-depmod/System.map-\$4" "\$4"
fi
EOF
chmod +x ${D}${bindir_crossscripts}/depmodwrapper
Does anything in the build install to $sysconfdir/depmod.d ?
Not that I know of, it's only created by the kmod recipe.
Sau!
Cheers,
Richard
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#163943):
https://lists.openembedded.org/g/openembedded-core/message/163943
Mute This Topic: https://lists.openembedded.org/mt/90166048/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 3/3] depmodwrapper: Use nonarch_base_libdir for depmod.d
This ensure that when depmod-native runs we can find the correct
exclude.conf information, in this case adding .debug to ignore
the .debug kernell modules. The kmod utilities like depmod can use
either /etc/depmod.d or /lib/depmod.d. The kmod recipe is installing
the existing search.conf to /lib/depmod.d (nonarch_base_lib)
When the busybox modutils are used, /lib/depmod.d is not used, so
it's safe add the exclude.conf file to /lib/depmod.d.
Signed-off-by: Saul Wold
---
meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb | 9 +++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
index 04fc14a6d21..65068f02df8 100644
--- a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
+++ b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
@@ -30,11 +30,16 @@ if [ -r "${PKGDATA_DIR}/kernel-depmod/kernel-abiversion" ];
then
kernelabi=\$(cat "${PKGDATA_DIR}/kernel-depmod/kernel-abiversion")
fi
+if [ ! -e "\3${nonarch_base_libdir}/depmod.d/exclude.conf" ]; then
+mkdir -p "\$3${nonarch_base_libdir}/depmod.d"
+echo "exclude .debug" > "\$3${nonarch_base_libdir}/depmod.d/exclude.conf"
+fi
+
if [ ! -r ${PKGDATA_DIR}/kernel-depmod/System.map-\$4 ] || [ "\$kernelabi" !=
"\$4" ]; then
echo "Unable to read: ${PKGDATA_DIR}/kernel-depmod/System.map-\$4" >&2
-exec env depmod -C "\$3${sysconfdir}/depmod.d" "\$1" "\$2" "\$3" "\$4"
+exec env depmod -C "\$3${nonarch_base_libdir}/depmod.d" "\$1" "\$2" "\$3"
"\$4"
else
-exec env depmod -C "\$3${sysconfdir}/depmod.d" "\$1" "\$2" "\$3" -F
"${PKGDATA_DIR}/kernel-depmod/System.map-\$4" "\$4"
+exec env depmod -C "\$3${nonarch_base_libdir}/depmod.d" "\$1" "\$2" "\$3"
-F "${PKGDATA_DIR}/kernel-depmod/System.map-\$4" "\$4"
fi
EOF
chmod +x ${D}${bindir_crossscripts}/depmodwrapper
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#163874):
https://lists.openembedded.org/g/openembedded-core/message/163874
Mute This Topic: https://lists.openembedded.org/mt/90166048/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 2/3] kmod: Add an exclude directive to depmod
This adds a new configuration directive to depmod that causes
depmod to exclude a give path entry like .debug.
kernel-dbg provides the modules .debug/.ko files and
when installed either directly or when dbg-pkgs are selected
this can cause depmod to fail.
Signed-off-by: Saul Wold
---
v2: Updated patch based on upstream comments
...dd-support-for-excluding-a-directory.patch | 169 ++
meta/recipes-kernel/kmod/kmod_29.bb | 4 +
2 files changed, 173 insertions(+)
create mode 100644
meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
diff --git
a/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
new file mode 100644
index 000..18d97935331
--- /dev/null
+++
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
@@ -0,0 +1,169 @@
+From 01f3fe68a7a42b06eb318f3b09fa5e5ea75d46c4 Mon Sep 17 00:00:00 2001
+From: Saul Wold
+Date: Tue, 22 Mar 2022 12:11:45 -0700
+Subject: [PATCH] depmod: Add support for excluding a directory
+
+This adds support to depmod to enable a new exclude directive in
+the depmod.d/*.conf configuration file. Currently depmod
+already excludes directories named source or build. This change
+will allow additional directories like .debug to be excluded also
+via a new exclude directive.
+
+depmod.d/exclude.conf example:
+exclude.debug
+
+Upstream-Status: Submitted
+
+Signed-off-by: Saul Wold
+---
+ man/depmod.d.xml | 14 +++
+ tools/depmod.c | 65 +---
+ 2 files changed, 75 insertions(+), 4 deletions(-)
+
+diff --git a/man/depmod.d.xml b/man/depmod.d.xml
+index b315e93..76548e9 100644
+--- a/man/depmod.d.xml
b/man/depmod.d.xml
+@@ -131,6 +131,20 @@
+
+
+
++
++exclude excludedir
++
++
++
++This specifies the trailing directories that will be excluded
++during the search for kernel modules.
++
++
++ The excludedir is the trailing directory
++ to exclude
++
++
++
+
+
+
+diff --git a/tools/depmod.c b/tools/depmod.c
+index eb810b8..ac365e9 100644
+--- a/tools/depmod.c
b/tools/depmod.c
+@@ -458,6 +458,11 @@ struct cfg_external {
+ char path[];
+ };
+
++struct cfg_exclude {
++ struct cfg_exclude *next;
++ char exclude_dir[];
++};
++
+ struct cfg {
+ const char *kversion;
+ char dirname[PATH_MAX];
+@@ -469,6 +474,7 @@ struct cfg {
+ struct cfg_override *overrides;
+ struct cfg_search *searches;
+ struct cfg_external *externals;
++ struct cfg_exclude *excludes;
+ };
+
+ static enum search_type cfg_define_search_type(const char *path)
+@@ -580,6 +586,30 @@ static void cfg_external_free(struct cfg_external *ext)
+ free(ext);
+ }
+
++static int cfg_exclude_add(struct cfg *cfg, const char *path)
++{
++ struct cfg_exclude *exc;
++ size_t len = strlen(path);
++
++ exc = malloc(sizeof(struct cfg_exclude) + len + 1);
++ if (exc == NULL) {
++ ERR("exclude add: out of memory\n");
++ return -ENOMEM;
++ }
++ memcpy(exc->exclude_dir, path, len + 1);
++
++ DBG("exclude add: %s\n", path);
++
++ exc->next = cfg->excludes;
++ cfg->excludes = exc;
++ return 0;
++}
++
++static void cfg_exclude_free(struct cfg_exclude *exc)
++{
++ free(exc);
++}
++
+ static int cfg_kernel_matches(const struct cfg *cfg, const char *pattern)
+ {
+ regex_t re;
+@@ -657,6 +687,11 @@ static int cfg_file_parse(struct cfg *cfg, const char
*filename)
+ }
+
+ cfg_external_add(cfg, dir);
++ } else if (streq(cmd, "exclude")) {
++ const char *sp;
++ while ((sp = strtok_r(NULL, "\t ", )) != NULL) {
++ cfg_exclude_add(cfg, sp);
++ }
+ } else if (streq(cmd, "include")
+ || streq(cmd, "make_map_files")) {
+ INF("%s:%u: command %s not implemented yet\n",
+@@ -857,6 +892,12 @@ static void cfg_free(struct cfg *cfg)
+ cfg->externals = cfg->externals->next;
+ cfg_external_free(tmp);
+ }
++
++ while (cfg->excludes) {
++ struct cfg_exclude *tmp = cfg->excludes;
++ cfg->excludes = cfg->excludes->next;
++ cfg_exclude_free(tmp);
++ }
+ }
+
+
+@@ -1229,6 +1270,24 @@ add:
+ return 0;
+ }
+
++static int should_exclude_dir(struct cfg *cfg, char *name)
++{
++ struct cfg_exclude *exc;
++
++ if (name[0] == '.' && (name[1] == '\0' ||
++
[OE-core] [PATCH v2 1/3] busybox: Exclude .debug from depmod
As with the kmod version of depmod, exclude .debug from being
searched. Since busybox does not use the depmod.d and any
configuration file option is ignored we just hardcode it.
Signed-off-by: Saul Wold
---
...001-depmod-Ignore-.debug-directories.patch | 32 +++
meta/recipes-core/busybox/busybox_1.35.0.bb | 1 +
2 files changed, 33 insertions(+)
create mode 100644
meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch
diff --git
a/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch
b/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch
new file mode 100644
index 000..354f83a4a5f
--- /dev/null
+++
b/meta/recipes-core/busybox/busybox/0001-depmod-Ignore-.debug-directories.patch
@@ -0,0 +1,32 @@
+From 5f6ed003f10ee0bd4a508d5f59129a29f0920dfc Mon Sep 17 00:00:00 2001
+From: Saul Wold
+Date: Thu, 31 Mar 2022 11:21:45 -0700
+Subject: [PATCH] depmod: Ignore .debug directories
+
+The .debug/.ko files do not have the correct symbol information
+since it's split away from the actual .ko file. Just ignore it.
+
+Upstream-Status: Pending
+
+Signed-off-by: Saul Wold
+---
+ modutils/depmod.c | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/modutils/depmod.c b/modutils/depmod.c
+index bb42bbe..aa5a2de 100644
+--- a/modutils/depmod.c
b/modutils/depmod.c
+@@ -43,6 +43,9 @@ static int FAST_FUNC parse_module(struct recursive_state
*state,
+ /* Arbitrary. Was sb->st_size, but that breaks .gz etc */
+ size_t len = (64*1024*1024 - 4096);
+
++ if (strstr(fname, ".debug") == NULL)
++ return TRUE;
++
+ if (strrstr(fname, ".ko") == NULL)
+ return TRUE;
+
+--
+2.31.1
+
diff --git a/meta/recipes-core/busybox/busybox_1.35.0.bb
b/meta/recipes-core/busybox/busybox_1.35.0.bb
index 7ce17170462..ab11f3d89a8 100644
--- a/meta/recipes-core/busybox/busybox_1.35.0.bb
+++ b/meta/recipes-core/busybox/busybox_1.35.0.bb
@@ -1,6 +1,7 @@
require busybox.inc
SRC_URI = "https://busybox.net/downloads/busybox-${PV}.tar.bz2;name=tarball \
+ file://0001-depmod-Ignore-.debug-directories.patch \
file://busybox-udhcpc-no_deconfig.patch \
file://find-touchscreen.sh \
file://busybox-cron \
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#163873):
https://lists.openembedded.org/g/openembedded-core/message/163873
Mute This Topic: https://lists.openembedded.org/mt/90166047/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 1/2] kmod: Add an exclude directive to depmod
I got some feedback from the kmod upstream, a v2 will be coming soon.
Sau!
On 3/30/22 15:11, Saul Wold wrote:
This adds a new configuration directive to depmod that causes
depmod to exclude a give path entry like .debug.
kernel-dbg provides the modules .debug/.ko files and
when installed either directly or when dbg-pkgs are selected
this can cause depmod to fail.
This patch will be submitted to upstream kmod.
Signed-off-by: Saul Wold
---
.../kmod/depmodwrapper-cross_1.0.bb | 3 +
...dd-support-for-excluding-a-directory.patch | 158 ++
meta/recipes-kernel/kmod/kmod_29.bb | 4 +
3 files changed, 165 insertions(+)
create mode 100644
meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
diff --git a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
index 04fc14a6d21..aa23ba41276 100644
--- a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
+++ b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
@@ -16,6 +16,9 @@ do_populate_sysroot[depends] = ""
do_install() {
install -d ${D}${bindir_crossscripts}/
+ install -d ${D}${sysconfdir}/depmod.d/
+
+ echo "exclude .debug" > ${D}${sysconfdir}/depmod.d/exclude.conf
cat > ${D}${bindir_crossscripts}/depmodwrapper << EOF
#!/bin/sh
diff --git
a/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
new file mode 100644
index 000..3f16cdf0574
--- /dev/null
+++
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
@@ -0,0 +1,158 @@
+From 8bc07c3ba3a412bd6bb94ad8bad0d76801ec2c9f Mon Sep 17 00:00:00 2001
+From: Saul Wold
+Date: Tue, 22 Mar 2022 12:11:45 -0700
+Subject: [PATCH] depmod: Add support for excluding a directory
+
+This adds support to depmod to enable a new exclude directive in
+the depmod.d/exclude.conf configuration file. Currently depmod
+already excludes directories named source or build. This change
+will allow additional directories like .debug to be excluded also
+via a new exclude directive.
+
+depmod.d/exclude.conf example:
+exclude.debug
+
+Upstream-Status: Submitted
+
+Signed-off-by: Saul Wold
+
+%% original patch: 0001-depmod-Add-support-for-excluding-a-directory.patch
+---
+ man/depmod.d.xml | 14 +
+ tools/depmod.c | 54
+ 2 files changed, 68 insertions(+)
+
+diff --git a/man/depmod.d.xml b/man/depmod.d.xml
+index b315e93..9ab790a 100644
+--- a/man/depmod.d.xml
b/man/depmod.d.xml
+@@ -131,6 +131,20 @@
+
+
+
++
++external excludedir
++
++
++
++This specifies the trailing directories that will be excluded
++during the search for kernel modules.
++
++
++ The excludedir the trailing directory
++ to exclude
++
++
++
+
+
+
+diff --git a/tools/depmod.c b/tools/depmod.c
+index eb810b8..8b19ab6 100644
+--- a/tools/depmod.c
b/tools/depmod.c
+@@ -458,6 +458,12 @@ struct cfg_external {
+ char path[];
+ };
+
++struct cfg_exclude {
++ struct cfg_exclude *next;
++ size_t len;
++ char exclude_dir[];
++};
++
+ struct cfg {
+ const char *kversion;
+ char dirname[PATH_MAX];
+@@ -469,6 +475,7 @@ struct cfg {
+ struct cfg_override *overrides;
+ struct cfg_search *searches;
+ struct cfg_external *externals;
++ struct cfg_exclude *excludes;
+ };
+
+ static enum search_type cfg_define_search_type(const char *path)
+@@ -580,6 +587,31 @@ static void cfg_external_free(struct cfg_external *ext)
+ free(ext);
+ }
+
++static int cfg_exclude_add(struct cfg *cfg, const char *path)
++{
++ struct cfg_exclude *exc;
++ size_t len = strlen(path);
++
++ exc = malloc(sizeof(struct cfg_exclude) + len);
++ if (exc == NULL) {
++ ERR("exclude add: out of memory\n");
++ return -ENOMEM;
++ }
++ exc->len = len;
++ memcpy(exc->exclude_dir, path, len);
++
++ DBG("exclude add: %s\n", path);
++
++ exc->next = cfg->excludes;
++ cfg->excludes = exc;
++ return 0;
++}
++
++static void cfg_exclude_free(struct cfg_exclude *exc)
++{
++ free(exc);
++}
++
+ static int cfg_kernel_matches(const struct cfg *cfg, const char *pattern)
+ {
+ regex_t re;
+@@ -657,6 +689,11 @@ static int cfg_file_parse(struct cfg *cfg, const char
*filename)
+ }
+
+ cfg_external_add(cfg, dir);
++ } else if (streq(cmd, "exclude")) {
++ const char *sp;
++ while ((sp = strtok_r(NULL,
[OE-core] [PATCH 2/2] depmodwrapper: Use native staging dir
Use the native staging dir so that we can get the correct depmod.d configuration
files. When depmod runs we want to ensure that the newly supported exclude.conf
is read so that .debug/.ko files are excluded.
Signed-off-by: Saul Wold
---
meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
index aa23ba41276..9921b7e8ad7 100644
--- a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
+++ b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
@@ -35,9 +35,9 @@ fi
if [ ! -r ${PKGDATA_DIR}/kernel-depmod/System.map-\$4 ] || [ "\$kernelabi" !=
"\$4" ]; then
echo "Unable to read: ${PKGDATA_DIR}/kernel-depmod/System.map-\$4" >&2
-exec env depmod -C "\$3${sysconfdir}/depmod.d" "\$1" "\$2" "\$3" "\$4"
+exec env depmod -C "\${STAGING_BASE_LIBDIR_NATIVE}/depmod.d" "\$1" "\$2"
"\$3" "\$4"
else
-exec env depmod -C "\$3${sysconfdir}/depmod.d" "\$1" "\$2" "\$3" -F
"${PKGDATA_DIR}/kernel-depmod/System.map-\$4" "\$4"
+exec env depmod -C "\${STAGING_BASE_LIBDIR_NATIVE}/depmod.d" "\$1" "\$2"
"\$3" -F "${PKGDATA_DIR}/kernel-depmod/System.map-\$4" "\$4"
fi
EOF
chmod +x ${D}${bindir_crossscripts}/depmodwrapper
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#163794):
https://lists.openembedded.org/g/openembedded-core/message/163794
Mute This Topic: https://lists.openembedded.org/mt/90143169/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/2] kmod: Add an exclude directive to depmod
This adds a new configuration directive to depmod that causes
depmod to exclude a give path entry like .debug.
kernel-dbg provides the modules .debug/.ko files and
when installed either directly or when dbg-pkgs are selected
this can cause depmod to fail.
This patch will be submitted to upstream kmod.
Signed-off-by: Saul Wold
---
.../kmod/depmodwrapper-cross_1.0.bb | 3 +
...dd-support-for-excluding-a-directory.patch | 158 ++
meta/recipes-kernel/kmod/kmod_29.bb | 4 +
3 files changed, 165 insertions(+)
create mode 100644
meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
diff --git a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
index 04fc14a6d21..aa23ba41276 100644
--- a/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
+++ b/meta/recipes-kernel/kmod/depmodwrapper-cross_1.0.bb
@@ -16,6 +16,9 @@ do_populate_sysroot[depends] = ""
do_install() {
install -d ${D}${bindir_crossscripts}/
+ install -d ${D}${sysconfdir}/depmod.d/
+
+ echo "exclude .debug" > ${D}${sysconfdir}/depmod.d/exclude.conf
cat > ${D}${bindir_crossscripts}/depmodwrapper << EOF
#!/bin/sh
diff --git
a/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
new file mode 100644
index 000..3f16cdf0574
--- /dev/null
+++
b/meta/recipes-kernel/kmod/kmod/0001-depmod-Add-support-for-excluding-a-directory.patch
@@ -0,0 +1,158 @@
+From 8bc07c3ba3a412bd6bb94ad8bad0d76801ec2c9f Mon Sep 17 00:00:00 2001
+From: Saul Wold
+Date: Tue, 22 Mar 2022 12:11:45 -0700
+Subject: [PATCH] depmod: Add support for excluding a directory
+
+This adds support to depmod to enable a new exclude directive in
+the depmod.d/exclude.conf configuration file. Currently depmod
+already excludes directories named source or build. This change
+will allow additional directories like .debug to be excluded also
+via a new exclude directive.
+
+depmod.d/exclude.conf example:
+exclude.debug
+
+Upstream-Status: Submitted
+
+Signed-off-by: Saul Wold
+
+%% original patch: 0001-depmod-Add-support-for-excluding-a-directory.patch
+---
+ man/depmod.d.xml | 14 +
+ tools/depmod.c | 54
+ 2 files changed, 68 insertions(+)
+
+diff --git a/man/depmod.d.xml b/man/depmod.d.xml
+index b315e93..9ab790a 100644
+--- a/man/depmod.d.xml
b/man/depmod.d.xml
+@@ -131,6 +131,20 @@
+
+
+
++
++external excludedir
++
++
++
++This specifies the trailing directories that will be excluded
++during the search for kernel modules.
++
++
++ The excludedir the trailing directory
++ to exclude
++
++
++
+
+
+
+diff --git a/tools/depmod.c b/tools/depmod.c
+index eb810b8..8b19ab6 100644
+--- a/tools/depmod.c
b/tools/depmod.c
+@@ -458,6 +458,12 @@ struct cfg_external {
+ char path[];
+ };
+
++struct cfg_exclude {
++ struct cfg_exclude *next;
++ size_t len;
++ char exclude_dir[];
++};
++
+ struct cfg {
+ const char *kversion;
+ char dirname[PATH_MAX];
+@@ -469,6 +475,7 @@ struct cfg {
+ struct cfg_override *overrides;
+ struct cfg_search *searches;
+ struct cfg_external *externals;
++ struct cfg_exclude *excludes;
+ };
+
+ static enum search_type cfg_define_search_type(const char *path)
+@@ -580,6 +587,31 @@ static void cfg_external_free(struct cfg_external *ext)
+ free(ext);
+ }
+
++static int cfg_exclude_add(struct cfg *cfg, const char *path)
++{
++ struct cfg_exclude *exc;
++ size_t len = strlen(path);
++
++ exc = malloc(sizeof(struct cfg_exclude) + len);
++ if (exc == NULL) {
++ ERR("exclude add: out of memory\n");
++ return -ENOMEM;
++ }
++ exc->len = len;
++ memcpy(exc->exclude_dir, path, len);
++
++ DBG("exclude add: %s\n", path);
++
++ exc->next = cfg->excludes;
++ cfg->excludes = exc;
++ return 0;
++}
++
++static void cfg_exclude_free(struct cfg_exclude *exc)
++{
++ free(exc);
++}
++
+ static int cfg_kernel_matches(const struct cfg *cfg, const char *pattern)
+ {
+ regex_t re;
+@@ -657,6 +689,11 @@ static int cfg_file_parse(struct cfg *cfg, const char
*filename)
+ }
+
+ cfg_external_add(cfg, dir);
++ } else if (streq(cmd, "exclude")) {
++ const char *sp;
++ while ((sp = strtok_r(NULL, "\t ", )) != NULL) {
++ cfg_exclude_add(cfg, sp);
++ }
+ } else
[OE-core] [PATCH v2] Change internal variables
Signed-off-by: Saul Wold
---
meta/classes/base.bbclass | 4 +--
meta/classes/cross-canadian.bbclass | 6 ++--
meta/classes/cve-check.bbclass| 31 ++-
meta/classes/insane.bbclass | 7 +++--
meta/classes/populate_sdk_ext.bbclass | 18 +--
meta/classes/sstate.bbclass | 4 +--
.../distro/include/cve-extra-exclusions.inc | 2 +-
meta/conf/distro/include/security_flags.inc | 2 +-
meta/lib/oe/utils.py | 4 +--
meta/lib/oeqa/manual/bsp-hw.json | 2 +-
.../lib/oeqa/selftest/cases/containerimage.py | 2 +-
scripts/lib/checklayer/cases/bsp.py | 2 +-
scripts/verify-bashisms | 10 +++---
13 files changed, 48 insertions(+), 46 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index b7869da3b3..cc81461473 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -329,9 +329,9 @@ python base_eventhandler() {
source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', False)
if not source_mirror_fetch:
provs = (d.getVar("PROVIDES") or "").split()
-multiwhitelist = (d.getVar("BB_MULTI_PROVIDER_ALLOWED") or
"").split()
+multiprovidersallowed = (d.getVar("BB_MULTI_PROVIDER_ALLOWED") or
"").split()
for p in provs:
-if p.startswith("virtual/") and p not in multiwhitelist:
+if p.startswith("virtual/") and p not in multiprovidersallowed:
profprov = d.getVar("PREFERRED_PROVIDER_" + p)
if profprov and pn != profprov:
raise bb.parse.SkipRecipe("PREFERRED_PROVIDER_%s set
to %s, not %s" % (p, profprov, pn))
diff --git a/meta/classes/cross-canadian.bbclass
b/meta/classes/cross-canadian.bbclass
index ac82e86356..a0e9d23836 100644
--- a/meta/classes/cross-canadian.bbclass
+++ b/meta/classes/cross-canadian.bbclass
@@ -36,7 +36,7 @@ python () {
return
tos = d.getVar("TARGET_OS")
-whitelist = ["mingw32"]
+tos_known = ["mingw32"]
extralibcs = [""]
if "musl" in d.getVar("BASECANADIANEXTRAOS"):
extralibcs.append("musl")
@@ -51,8 +51,8 @@ python () {
entry = entry + "-gnu" + variant
elif libc:
entry = entry + "-" + libc
-whitelist.append(entry)
-if tos not in whitelist:
+tos_known.append(entry)
+if tos not in tos_known:
bb.fatal("Building cross-candian for an unknown TARGET_SYS (%s),
please update cross-canadian.bbclass" % d.getVar("TARGET_SYS"))
for n in ["PROVIDES", "DEPENDS"]:
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 079d09a76f..dfad10c22b 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -43,11 +43,12 @@ CVE_CHECK_CREATE_MANIFEST ??= "1"
CVE_CHECK_REPORT_PATCHED ??= "1"
-# Whitelist for packages (PN)
+# Skip CVE Check for packages (PN)
CVE_CHECK_SKIP_RECIPE ?= ""
-# Whitelist for CVE. If a CVE is found, then it is considered patched.
-# The value is a string containing space separated CVE values:
+# Ingore the check for a given list of CVEs. If a CVE is found,
+# then it is considered patched. The value is a string containing
+# space separated CVE values:
#
# CVE_CHECK_IGNORE = 'CVE-2014-2524 CVE-2018-1234'
#
@@ -101,10 +102,10 @@ python do_cve_check () {
patched_cves = get_patched_cves(d)
except FileNotFoundError:
bb.fatal("Failure in searching patches")
-whitelisted, patched, unpatched = check_cves(d, patched_cves)
+ignored, patched, unpatched = check_cves(d, patched_cves)
if patched or unpatched:
cve_data = get_cve_info(d, patched + unpatched)
-cve_write_data(d, patched, unpatched, whitelisted, cve_data)
+cve_write_data(d, patched, unpatched, ignored, cve_data)
else:
bb.note("No CVE database found, skipping CVE check")
@@ -176,12 +177,12 @@ def check_cves(d, patched_cves):
return ([], [], [])
pv = d.getVar("CVE_VERSION").split("+git")[0]
-# If the recipe has been whitelisted we return empty lists
+# If the recipe has been skipped/ignored we return empty lists
if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split():
-bb.note("Recipe has been whitelisted, skipping check")
+bb.note("Recipe has been skipped by cve-check")
return ([], [], [])
-cve_whitelist = d.getVar("CVE_CHECK_IGNORE").split()
+cve
[OE-core] [PATCH] Change internal variables
This patch is another part of the inclusive language change
to rename internal variables to more understandable names.
Signed-off-by: Saul Wold
---
meta/classes/base.bbclass | 4 +--
meta/classes/cross-canadian.bbclass | 6 ++--
meta/classes/cve-check.bbclass| 31 ++-
meta/classes/insane.bbclass | 6 ++--
meta/classes/populate_sdk_ext.bbclass | 18 +--
meta/classes/sstate.bbclass | 4 +--
.../distro/include/cve-extra-exclusions.inc | 2 +-
meta/conf/distro/include/security_flags.inc | 2 +-
meta/lib/oe/utils.py | 4 +--
meta/lib/oeqa/manual/bsp-hw.json | 2 +-
.../lib/oeqa/selftest/cases/containerimage.py | 2 +-
scripts/lib/checklayer/cases/bsp.py | 2 +-
scripts/verify-bashisms | 10 +++---
13 files changed, 47 insertions(+), 46 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index b7869da3b3..cc81461473 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -329,9 +329,9 @@ python base_eventhandler() {
source_mirror_fetch = d.getVar('SOURCE_MIRROR_FETCH', False)
if not source_mirror_fetch:
provs = (d.getVar("PROVIDES") or "").split()
-multiwhitelist = (d.getVar("BB_MULTI_PROVIDER_ALLOWED") or
"").split()
+multiprovidersallowed = (d.getVar("BB_MULTI_PROVIDER_ALLOWED") or
"").split()
for p in provs:
-if p.startswith("virtual/") and p not in multiwhitelist:
+if p.startswith("virtual/") and p not in multiprovidersallowed:
profprov = d.getVar("PREFERRED_PROVIDER_" + p)
if profprov and pn != profprov:
raise bb.parse.SkipRecipe("PREFERRED_PROVIDER_%s set
to %s, not %s" % (p, profprov, pn))
diff --git a/meta/classes/cross-canadian.bbclass
b/meta/classes/cross-canadian.bbclass
index ac82e86356..a0e9d23836 100644
--- a/meta/classes/cross-canadian.bbclass
+++ b/meta/classes/cross-canadian.bbclass
@@ -36,7 +36,7 @@ python () {
return
tos = d.getVar("TARGET_OS")
-whitelist = ["mingw32"]
+tos_known = ["mingw32"]
extralibcs = [""]
if "musl" in d.getVar("BASECANADIANEXTRAOS"):
extralibcs.append("musl")
@@ -51,8 +51,8 @@ python () {
entry = entry + "-gnu" + variant
elif libc:
entry = entry + "-" + libc
-whitelist.append(entry)
-if tos not in whitelist:
+tos_known.append(entry)
+if tos not in tos_known:
bb.fatal("Building cross-candian for an unknown TARGET_SYS (%s),
please update cross-canadian.bbclass" % d.getVar("TARGET_SYS"))
for n in ["PROVIDES", "DEPENDS"]:
diff --git a/meta/classes/cve-check.bbclass b/meta/classes/cve-check.bbclass
index 079d09a76f..2dad58c13e 100644
--- a/meta/classes/cve-check.bbclass
+++ b/meta/classes/cve-check.bbclass
@@ -43,11 +43,12 @@ CVE_CHECK_CREATE_MANIFEST ??= "1"
CVE_CHECK_REPORT_PATCHED ??= "1"
-# Whitelist for packages (PN)
+# Skip CVE Check for packages (PN)
CVE_CHECK_SKIP_RECIPE ?= ""
-# Whitelist for CVE. If a CVE is found, then it is considered patched.
-# The value is a string containing space separated CVE values:
+# Ingore the check for a given list of CVEs. If a CVE is found,
+# then it is considered patched. The value is a string containing
+# space separated CVE values:
#
# CVE_CHECK_IGNORE = 'CVE-2014-2524 CVE-2018-1234'
#
@@ -101,10 +102,10 @@ python do_cve_check () {
patched_cves = get_patched_cves(d)
except FileNotFoundError:
bb.fatal("Failure in searching patches")
-whitelisted, patched, unpatched = check_cves(d, patched_cves)
+ignored, patched, unpatched = check_cves(d, patched_cves)
if patched or unpatched:
cve_data = get_cve_info(d, patched + unpatched)
-cve_write_data(d, patched, unpatched, whitelisted, cve_data)
+cve_write_data(d, patched, unpatched, ignored, cve_data)
else:
bb.note("No CVE database found, skipping CVE check")
@@ -176,12 +177,12 @@ def check_cves(d, patched_cves):
return ([], [], [])
pv = d.getVar("CVE_VERSION").split("+git")[0]
-# If the recipe has been whitelisted we return empty lists
+# If the recipe has been skipped/ignored we return empty lists
if pn in d.getVar("CVE_CHECK_SKIP_RECIPE").split():
-bb.note("Recipe has been whitelisted, skipping check")
+bb.note("Recipe has been skipped/ignored, skippin
[OE-core] [PATCH] convert-variable-renames: Fix output string
Signed-off-by: Saul Wold
---
scripts/contrib/convert-variable-renames.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/contrib/convert-variable-renames.py
b/scripts/contrib/convert-variable-renames.py
index ed012610db..856c001e11 100755
--- a/scripts/contrib/convert-variable-renames.py
+++ b/scripts/contrib/convert-variable-renames.py
@@ -79,7 +79,7 @@ def processfile(fn):
# Find removed names
for removed_name in removed_list:
if removed_name in line:
-print("%s needs further work at line %s because
has been deprecated" % (fn, lineno, remove_name))
+print("%s needs further work at line %s because %s
has been deprecated" % (fn, lineno, removed_name))
for check_word in context_check_list:
if re.search(check_word, line, re.IGNORECASE):
print("%s needs further work at line %s since it
contains %s"% (fn, lineno, check_word))
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#162632):
https://lists.openembedded.org/g/openembedded-core/message/162632
Mute This Topic: https://lists.openembedded.org/mt/89515823/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v3 2/2] license.py: rename variales
Signed-off-by: Saul Wold --- meta/lib/oe/license.py | 28 +++- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index b5d378a549b..4cd382b4fd7 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -99,20 +99,22 @@ def flattened_licenses(licensestr, choose_licenses): raise LicenseSyntaxError(licensestr, exc) return flatten.licenses -def is_included(licensestr, whitelist=None, blacklist=None): -"""Given a license string and whitelist and blacklist, determine if the -license string matches the whitelist and does not match the blacklist. - -Returns a tuple holding the boolean state and a list of the applicable -licenses that were excluded if state is False, or the licenses that were -included if the state is True. +def is_included(licensestr, include=None, exclude=None): +"""Given a license string and include list and exclude list, +determine if the license string matches the an included +license and does dont match an excluded license. + +Returns a tuple holding the boolean state and a list of +the applicable licenses that were excluded if state is +False, or the licenses that were included if the state +is True. """ def include_license(license): -return any(fnmatch(license, pattern) for pattern in whitelist) +return any(fnmatch(license, pattern) for pattern in include) def exclude_license(license): -return any(fnmatch(license, pattern) for pattern in blacklist) +return any(fnmatch(license, pattern) for pattern in exclude) def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most @@ -131,11 +133,11 @@ def is_included(licensestr, whitelist=None, blacklist=None): else: return beta -if not whitelist: -whitelist = ['*'] +if not include: +include = ['*'] -if not blacklist: -blacklist = [] +if not exclude: +exclude = [] licenses = flattened_licenses(licensestr, choose_licenses) excluded = [lic for lic in licenses if exclude_license(lic)] -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#162417): https://lists.openembedded.org/g/openembedded-core/message/162417 Mute This Topic: https://lists.openembedded.org/mt/89402861/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v3 1/2] INCOMPATIBLE_LICENSE re-work
From: Saul Wold
This re-writes the INCOMPATIBLE_LICENSE checking code to replace
the WHITELIST_ with
INCOMPATIBLE_LICENSE_EXCEPTIONS = ': : ...'
This initial set of changes leaves most of the code structure in
place, but the code in base.bbclass needs to be re-written to make
the check more consistent around packages (PKGS) and not recipe
names (PN). This also is taking into account the changes for SPDX
licenses.
Signed-off-by: Saul Wold
Signed-off-by: Richard Purdie
---
meta/classes/base.bbclass | 26 +-
meta/classes/license_image.bbclass| 27 +++
meta/classes/multilib.bbclass | 6 ++---
meta/conf/bitbake.conf| 10 +++
.../distro/include/default-distrovars.inc | 2 +-
.../oeqa/selftest/cases/incompatible_lic.py | 10 +++
6 files changed, 43 insertions(+), 38 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 55f654d37d0..ddca87d4a8c 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -595,21 +595,23 @@ python () {
if check_license and bad_licenses:
bad_licenses = expand_wildcard_licenses(d, bad_licenses)
-whitelist = []
-for lic in bad_licenses:
-spdx_license = return_spdx(d, lic)
-whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
-if spdx_license:
-whitelist.extend((d.getVar("WHITELIST_" + spdx_license) or
"").split())
-
-if pn in whitelist:
+exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or
"").split()
+
+pkg_exceptions = {}
+for exception in exceptions:
+pkg_lic = exception.split(':')
+pkg_exceptions[pkg_lic[0]] = pkg_lic[1]
+
+#if any((pn in execption and incompatible_lic in exception) for
execption in exceptions):
+if any(execption.startswith(pn + ':') for execption in exceptions):
'''
-We need to track what we are whitelisting and why. If pn is
-incompatible we need to be able to note that the image that
-is created may infact contain incompatible licenses despite
+We need to track which recipes are in the exception
+list and why. If pn is incompatible we need to be
+able to note that the image that is created may
+infact contain incompatible licenses despite
INCOMPATIBLE_LICENSE being set.
'''
-bb.note("Including %s as buildable despite it having an
incompatible license because it has been whitelisted" % pn)
+bb.note("Including %s as a buildable recipe despite it having
an incompatible license because it was found in the exception list" % pn)
else:
pkgs = d.getVar('PACKAGES').split()
skipped_pkgs = {}
diff --git a/meta/classes/license_image.bbclass
b/meta/classes/license_image.bbclass
index bf70bee99bb..c6f04d30733 100644
--- a/meta/classes/license_image.bbclass
+++ b/meta/classes/license_image.bbclass
@@ -54,28 +54,21 @@ def write_license_files(d, license_manifest, pkg_dic,
rootfs=True):
bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
bad_licenses = expand_wildcard_licenses(d, bad_licenses)
-whitelist = []
-for lic in bad_licenses:
-whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
-
+exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
with open(license_manifest, "w") as license_file:
for pkg in sorted(pkg_dic):
-if bad_licenses and pkg not in whitelist:
-try:
+if bad_licenses and not any((pkg + ":") in execption for execption
in exceptions):
licenses = incompatible_pkg_license(d, bad_licenses,
pkg_dic[pkg]["LICENSE"])
if licenses:
bb.fatal("Package %s cannot be installed into the
image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses)))
-(pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
-oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
-bad_licenses, canonical_license, d)
-except oe.license.LicenseError as exc:
-bb.fatal('%s: %s' % (d.getVar('P'), exc))
-else:
-pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ',
pkg_dic[pkg]["LICENSE"])
-pkg_dic[pkg]["LICENSES"] = re.sub(r' *', ' ',
pkg_dic[pkg]["LIC
[OE-core] [PATCH v3 2/2] INCOMPATIBLE_LICENSE: add has_pkg_license_exception()
This adds in the new function to check for both package and
license are in the new INCOMPATIBLE_LICENSE_EXCEPTION list.
This has been tested by changing the skeleton/hello to MIT-X
and using that license to verify it will be skipped or not
installed. oe-selftest was also used.
Signed-off-by: Saul Wold
Signed-off-by: Richard Purdie
---
meta/classes/base.bbclass | 66 --
meta/classes/license_image.bbclass | 5 ++-
meta/lib/oe/license.py | 10 +
3 files changed, 40 insertions(+), 41 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index ddca87d4a8c..fccf3df17ff 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -597,46 +597,34 @@ python () {
exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or
"").split()
-pkg_exceptions = {}
-for exception in exceptions:
-pkg_lic = exception.split(':')
-pkg_exceptions[pkg_lic[0]] = pkg_lic[1]
-
-#if any((pn in execption and incompatible_lic in exception) for
execption in exceptions):
-if any(execption.startswith(pn + ':') for execption in exceptions):
-'''
-We need to track which recipes are in the exception
-list and why. If pn is incompatible we need to be
-able to note that the image that is created may
-infact contain incompatible licenses despite
-INCOMPATIBLE_LICENSE being set.
-'''
-bb.note("Including %s as a buildable recipe despite it having
an incompatible license because it was found in the exception list" % pn)
-else:
-pkgs = d.getVar('PACKAGES').split()
-skipped_pkgs = {}
-unskipped_pkgs = []
-for pkg in pkgs:
-incompatible_lic = incompatible_license(d, bad_licenses,
pkg)
-if incompatible_lic:
-skipped_pkgs[pkg] = incompatible_lic
-else:
-unskipped_pkgs.append(pkg)
-if unskipped_pkgs:
-for pkg in skipped_pkgs:
-bb.debug(1, "Skipping the package %s at do_rootfs
because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
-d.setVar('_exclude_incompatible-' + pkg, '
'.join(skipped_pkgs[pkg]))
-for pkg in unskipped_pkgs:
-bb.debug(1, "Including the package %s" % pkg)
+pkgs = d.getVar('PACKAGES').split()
+skipped_pkgs = {}
+unskipped_pkgs = []
+for pkg in pkgs:
+pkg_exception = oe.license.has_pkg_license_exception(pkg,
bad_licenses, exceptions)
+
+incompatible_lic = incompatible_license(d, bad_licenses, pkg)
+if incompatible_lic and not pkg_exception:
+skipped_pkgs[pkg] = incompatible_lic
else:
-incompatible_lic = incompatible_license(d, bad_licenses)
-for pkg in skipped_pkgs:
-incompatible_lic += skipped_pkgs[pkg]
-incompatible_lic = sorted(list(set(incompatible_lic)))
-
-if incompatible_lic:
-bb.debug(1, "Skipping recipe %s because of
incompatible license(s): %s" % (pn, ' '.join(incompatible_lic)))
-raise bb.parse.SkipRecipe("it has incompatible
license(s): %s" % ' '.join(incompatible_lic))
+unskipped_pkgs.append(pkg)
+
+if unskipped_pkgs:
+for pkg in skipped_pkgs:
+bb.warn( "Skipping the package %s at do_rootfs because of
incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
+bb.debug(1, "Skipping the package %s at do_rootfs because
of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
+d.setVar('_exclude_incompatible-' + pkg, '
'.join(skipped_pkgs[pkg]))
+for pkg in unskipped_pkgs:
+bb.debug(1, "Including the package %s" % pkg)
+else:
+incompatible_lic = incompatible_license(d, bad_licenses)
+for pkg in skipped_pkgs:
+incompatible_lic += skipped_pkgs[pkg]
+incompatible_lic = sorted(list(set(incompatible_lic)))
+
+if incompatible_lic:
+bb.warn( "Skipping recipe %s because of incompatible
license(s): %s" % (pn, ' '.join(incompatible_lic)))
+raise bb.parse.SkipRecipe("it has incompatible license(s):
%s" % ' '.join(incompatible_lic))
needsrcrev = False
srcuri = d.g
[OE-core] [PATCH v2] license.py: rename variables
Update the comment to reflect new variable names Signed-off-by: Saul Wold --- v2: Update comment and change include -> include_licenses, exclude -> exclude_licenses meta/lib/oe/license.py | 31 +-- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index b5d378a549b..b1105f6149c 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -99,26 +99,29 @@ def flattened_licenses(licensestr, choose_licenses): raise LicenseSyntaxError(licensestr, exc) return flatten.licenses -def is_included(licensestr, whitelist=None, blacklist=None): -"""Given a license string and whitelist and blacklist, determine if the -license string matches the whitelist and does not match the blacklist. - -Returns a tuple holding the boolean state and a list of the applicable -licenses that were excluded if state is False, or the licenses that were -included if the state is True. +def is_included(licensestr, include_licenses=None, exclude_licenses=None): +"""Given a license a list of list to include and a list of +licenses to exclude, determine if the license string +matches the an include list and does not match the +exclude list. + +Returns a tuple holding the boolean state and a list of +the applicable licenses that were excluded if state is +False, or the licenses that were included if the state +is True. """ def include_license(license): -return any(fnmatch(license, pattern) for pattern in whitelist) +return any(fnmatch(license, pattern) for pattern in include_licenses) def exclude_license(license): -return any(fnmatch(license, pattern) for pattern in blacklist) +return any(fnmatch(license, pattern) for pattern in exclude_licenses) def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most included licenses and no excluded licenses).""" # The factor 1000 below is arbitrary, just expected to be much larger -# that the number of licenses actually specified. That way the weight +# than the number of licenses actually specified. That way the weight # will be negative if the list of licenses contains an excluded license, # but still gives a higher weight to the list with the most included # licenses. @@ -131,11 +134,11 @@ def is_included(licensestr, whitelist=None, blacklist=None): else: return beta -if not whitelist: -whitelist = ['*'] +if not include_licenses: +include = ['*'] -if not blacklist: -blacklist = [] +if not exclude_licenses: +exclude = [] licenses = flattened_licenses(licensestr, choose_licenses) excluded = [lic for lic in licenses if exclude_license(lic)] -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#162331): https://lists.openembedded.org/g/openembedded-core/message/162331 Mute This Topic: https://lists.openembedded.org/mt/89376290/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] license.py: rename variables
Update the comment to reflect new variable names Signed-off-by: Saul Wold --- meta/lib/oe/license.py | 28 +++- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py index b5d378a549b..4cd382b4fd7 100644 --- a/meta/lib/oe/license.py +++ b/meta/lib/oe/license.py @@ -99,20 +99,22 @@ def flattened_licenses(licensestr, choose_licenses): raise LicenseSyntaxError(licensestr, exc) return flatten.licenses -def is_included(licensestr, whitelist=None, blacklist=None): -"""Given a license string and whitelist and blacklist, determine if the -license string matches the whitelist and does not match the blacklist. - -Returns a tuple holding the boolean state and a list of the applicable -licenses that were excluded if state is False, or the licenses that were -included if the state is True. +def is_included(licensestr, include=None, exclude=None): +"""Given a license string and include list and exclude list, +determine if the license string matches the an included +license and does dont match an excluded license. + +Returns a tuple holding the boolean state and a list of +the applicable licenses that were excluded if state is +False, or the licenses that were included if the state +is True. """ def include_license(license): -return any(fnmatch(license, pattern) for pattern in whitelist) +return any(fnmatch(license, pattern) for pattern in include) def exclude_license(license): -return any(fnmatch(license, pattern) for pattern in blacklist) +return any(fnmatch(license, pattern) for pattern in exclude) def choose_licenses(alpha, beta): """Select the option in an OR which is the 'best' (has the most @@ -131,11 +133,11 @@ def is_included(licensestr, whitelist=None, blacklist=None): else: return beta -if not whitelist: -whitelist = ['*'] +if not include: +include = ['*'] -if not blacklist: -blacklist = [] +if not exclude: +exclude = [] licenses = flattened_licenses(licensestr, choose_licenses) excluded = [lic for lic in licenses if exclude_license(lic)] -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#162323): https://lists.openembedded.org/g/openembedded-core/message/162323 Mute This Topic: https://lists.openembedded.org/mt/89367148/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC] INCOMPATIBLE_LICENSE re-work
This re-writes the INCOMPATIBLE_LICENSE checking code to replace
the WHITELIST_ with
INCOMPATIBLE_LICENSE_EXCEPTIONS = ': : ...'
This initial set of changes leaves most of the code structure in
place, but the code in base.bbclass needs to be re-written to make
the check more consistent around packages (PKGS) and not recipe
names (PN). This also is taking into account the changes for SPDX
licenses.
This is a work in progress. This version does test successfully
with oe-selftest. This will be refactored as multiple patches as
appropriate.
Signed-off-by: Saul Wold
---
meta/classes/base.bbclass | 26 +-
meta/classes/license_image.bbclass| 27 +++
meta/classes/multilib.bbclass | 6 ++---
meta/conf/bitbake.conf| 10 +++
.../distro/include/default-distrovars.inc | 2 +-
.../oeqa/selftest/cases/incompatible_lic.py | 10 +++
6 files changed, 43 insertions(+), 38 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 227f1f5a756..f937728ef6a 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -595,21 +595,23 @@ python () {
if check_license and bad_licenses:
bad_licenses = expand_wildcard_licenses(d, bad_licenses)
-whitelist = []
-for lic in bad_licenses:
-spdx_license = return_spdx(d, lic)
-whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
-if spdx_license:
-whitelist.extend((d.getVar("WHITELIST_" + spdx_license) or
"").split())
-
-if pn in whitelist:
+exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or
"").split()
+
+pkg_exceptions = {}
+for exception in exceptions:
+pkg_lic = exception.split(':')
+pkg_exceptions[pkg_lic[0]] = pkg_lic[1]
+
+#if any((pn in execption and incompatible_lic in exception) for
execption in exceptions):
+if any(execption.startswith(pn + ':') for execption in exceptions):
'''
-We need to track what we are whitelisting and why. If pn is
-incompatible we need to be able to note that the image that
-is created may infact contain incompatible licenses despite
+We need to track which recipes are in the exception
+list and why. If pn is incompatible we need to be
+able to note that the image that is created may
+infact contain incompatible licenses despite
INCOMPATIBLE_LICENSE being set.
'''
-bb.note("Including %s as buildable despite it having an
incompatible license because it has been whitelisted" % pn)
+bb.note("Including %s as a buildable recipe despite it having
an incompatible license because it was found in the exception list" % pn)
else:
pkgs = d.getVar('PACKAGES').split()
skipped_pkgs = {}
diff --git a/meta/classes/license_image.bbclass
b/meta/classes/license_image.bbclass
index bf70bee99bb..c6f04d30733 100644
--- a/meta/classes/license_image.bbclass
+++ b/meta/classes/license_image.bbclass
@@ -54,28 +54,21 @@ def write_license_files(d, license_manifest, pkg_dic,
rootfs=True):
bad_licenses = (d.getVar("INCOMPATIBLE_LICENSE") or "").split()
bad_licenses = expand_wildcard_licenses(d, bad_licenses)
-whitelist = []
-for lic in bad_licenses:
-whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
-
+exceptions = (d.getVar("INCOMPATIBLE_LICENSE_EXCEPTIONS") or "").split()
with open(license_manifest, "w") as license_file:
for pkg in sorted(pkg_dic):
-if bad_licenses and pkg not in whitelist:
-try:
+if bad_licenses and not any((pkg + ":") in execption for execption
in exceptions):
licenses = incompatible_pkg_license(d, bad_licenses,
pkg_dic[pkg]["LICENSE"])
if licenses:
bb.fatal("Package %s cannot be installed into the
image because it has incompatible license(s): %s" %(pkg, ' '.join(licenses)))
-(pkg_dic[pkg]["LICENSE"], pkg_dic[pkg]["LICENSES"]) = \
-oe.license.manifest_licenses(pkg_dic[pkg]["LICENSE"],
-bad_licenses, canonical_license, d)
-except oe.license.LicenseError as exc:
-bb.fatal('%s: %s' % (d.getVar('P'), exc))
-else:
-pkg_dic[pkg]["LICENSES"] = re.sub(r'[|&()*]', ' ',
pkg_d
[OE-core] [PATCH] package: rename LICENSE_EXCLUSION
By renaming LICENSE_EXCLUSION to _exclude_incompatible, it makes it
clear that this is an internal variable.
Signed-off-by: Saul Wold
---
meta/classes/base.bbclass| 2 +-
meta/classes/package.bbclass | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 227f1f5a756..d0e669db00d 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -623,7 +623,7 @@ python () {
if unskipped_pkgs:
for pkg in skipped_pkgs:
bb.debug(1, "Skipping the package %s at do_rootfs
because of incompatible license(s): %s" % (pkg, ' '.join(skipped_pkgs[pkg])))
-d.setVar('LICENSE_EXCLUSION-' + pkg, '
'.join(skipped_pkgs[pkg]))
+d.setVar('_exclude_incompatible-' + pkg, '
'.join(skipped_pkgs[pkg]))
for pkg in unskipped_pkgs:
bb.debug(1, "Including the package %s" % pkg)
else:
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index f4a661ba25a..f8222581502 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1468,10 +1468,10 @@ python populate_packages () {
os.umask(oldumask)
os.chdir(workdir)
-# Handle LICENSE_EXCLUSION
+# Handle excluding packages with incompatible licenses
package_list = []
for pkg in packages:
-licenses = d.getVar('LICENSE_EXCLUSION-' + pkg)
+licenses = d.getVar('_exclude_incompatible-' + pkg)
if licenses:
msg = "Excluding %s from packaging as it has incompatible
license(s): %s" % (pkg, licenses)
oe.qa.handle_error("incompatible-license", msg, d)
@@ -2353,7 +2353,7 @@ def gen_packagevar(d, pkgvars="PACKAGEVARS"):
# Ensure that changes to INCOMPATIBLE_LICENSE re-run do_package for
# affected recipes.
-ret.append('LICENSE_EXCLUSION-%s' % p)
+ret.append('_exclude_incompatible-%s' % p)
return " ".join(ret)
PACKAGE_PREPROCESS_FUNCS ?= ""
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#162226):
https://lists.openembedded.org/g/openembedded-core/message/162226
Mute This Topic: https://lists.openembedded.org/mt/89324810/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] Rename LICENSE_FLAGS variable
From: Saul Wold
(From meta-yocto rev: e937a42996c046baca7ce502c6ce0ee3c7ed38e3)
Signed-off-by: Saul Wold
---
v2: rename internal variables and fix comments and messages
meta-poky/conf/local.conf.sample.extended | 2 +-
meta/classes/base.bbclass | 4 +--
meta/classes/license.bbclass | 35 ---
3 files changed, 21 insertions(+), 20 deletions(-)
diff --git a/meta-poky/conf/local.conf.sample.extended
b/meta-poky/conf/local.conf.sample.extended
index 1e3699ef8e6..bc2dec9f528 100644
--- a/meta-poky/conf/local.conf.sample.extended
+++ b/meta-poky/conf/local.conf.sample.extended
@@ -177,7 +177,7 @@ DISTRO_FEATURES:remove = "x11"
# product. If shipped as part of an image these packages may have
# implications so they are disabled by default. To enable them,
# un-comment the below as appropriate.
-#LICENSE_FLAGS_WHITELIST = "commercial_gst-fluendo-mp3 \
+#LICENSE_FLAGS_ACCEPTED = "commercial_gst-fluendo-mp3 \
# commercial_gst-openmax \
# commercial_gst-plugins-ugly \
# commercial_lame \
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index be820ddb2c2..227f1f5a756 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -542,9 +542,9 @@ python () {
unmatched_license_flags = check_license_flags(d)
if unmatched_license_flags:
if len(unmatched_license_flags) == 1:
-message = "because it has a restricted license '{0}'. Which is
not whitelisted in LICENSE_FLAGS_ACCEPTED".format(unmatched_license_flags[0])
+message = "because it has a restricted license '{0}'. Which is
not listed in LICENSE_FLAGS_ACCEPTED".format(unmatched_license_flags[0])
else:
-message = "because it has restricted licenses {0}. Which are
not whitelisted in LICENSE_FLAGS_ACCEPTED".format(
+message = "because it has restricted licenses {0}. Which are
not listed in LICENSE_FLAGS_ACCEPTED".format(
", ".join("'{0}'".format(f) for f in
unmatched_license_flags))
bb.debug(1, "Skipping %s %s" % (pn, message))
raise bb.parse.SkipRecipe(message)
diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index dd1e07ee377..dec98672096 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -341,30 +341,31 @@ def incompatible_license(d, dont_want_licenses,
package=None):
def check_license_flags(d):
"""
This function checks if a recipe has any LICENSE_FLAGS that
-aren't whitelisted.
+aren't acceptable.
-If it does, it returns the all LICENSE_FLAGS missing from the whitelist, or
-all of the LICENSE_FLAGS if there is no whitelist.
+If it does, it returns the all LICENSE_FLAGS missing from the list
+of acceptable license flags, or all of the LICENSE_FLAGS if there
+is no list of acceptable flags.
-If everything is is properly whitelisted, it returns None.
+If everything is is acceptable, it returns None.
"""
-def license_flag_matches(flag, whitelist, pn):
+def license_flag_matches(flag, acceptlist, pn):
"""
-Return True if flag matches something in whitelist, None if not.
+Return True if flag matches something in acceptlist, None if not.
-Before we test a flag against the whitelist, we append _${PN}
+Before we test a flag against the acceptlist, we append _${PN}
to it. We then try to match that string against the
-whitelist. This covers the normal case, where we expect
+acceptlist. This covers the normal case, where we expect
LICENSE_FLAGS to be a simple string like 'commercial', which
-the user typically matches exactly in the whitelist by
+the user typically matches exactly in the acceptlist by
explicitly appending the package name e.g 'commercial_foo'.
If we fail the match however, we then split the flag across
'_' and append each fragment and test until we either match or
run out of fragments.
"""
flag_pn = ("%s_%s" % (flag, pn))
-for candidate in whitelist:
+for candidate in acceptlist:
if flag_pn == candidate:
return True
@@ -375,27 +376,27 @@ def check_license_flags(d):
if flag_cur:
flag_cur += "_"
flag_cur += flagment
-for candidate in whitelist:
+for candidate in acceptlist:
if flag_cur == candidate:
return True
return False
-def all_license_flags_match(license_flags, whitelist):
+def all_license_flags_match(license_flags, acceptlist)
[OE-core] [PATCH] Rename LICENSE_FLAGS variable
Signed-off-by: Saul Wold
---
meta-poky/conf/local.conf.sample.extended| 2 +-
meta/classes/base.bbclass| 4 ++--
meta/classes/license.bbclass | 2 +-
meta/conf/bitbake.conf | 1 +
meta/lib/oeqa/selftest/cases/distrodata.py | 4 ++--
meta/lib/oeqa/selftest/cases/reproducible.py | 2 +-
meta/recipes-multimedia/gstreamer/gst-examples_1.18.6.bb | 4 ++--
scripts/contrib/convert-variable-renames.py | 1 +
8 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/meta-poky/conf/local.conf.sample.extended
b/meta-poky/conf/local.conf.sample.extended
index 1e3699ef8e6..bc2dec9f528 100644
--- a/meta-poky/conf/local.conf.sample.extended
+++ b/meta-poky/conf/local.conf.sample.extended
@@ -177,7 +177,7 @@ DISTRO_FEATURES:remove = "x11"
# product. If shipped as part of an image these packages may have
# implications so they are disabled by default. To enable them,
# un-comment the below as appropriate.
-#LICENSE_FLAGS_WHITELIST = "commercial_gst-fluendo-mp3 \
+#LICENSE_FLAGS_ACCEPTED = "commercial_gst-fluendo-mp3 \
# commercial_gst-openmax \
# commercial_gst-plugins-ugly \
# commercial_lame \
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 87a4cb5fc77..be820ddb2c2 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -542,9 +542,9 @@ python () {
unmatched_license_flags = check_license_flags(d)
if unmatched_license_flags:
if len(unmatched_license_flags) == 1:
-message = "because it has a restricted license '{0}'. Which is
not whitelisted in LICENSE_FLAGS_WHITELIST".format(unmatched_license_flags[0])
+message = "because it has a restricted license '{0}'. Which is
not whitelisted in LICENSE_FLAGS_ACCEPTED".format(unmatched_license_flags[0])
else:
-message = "because it has restricted licenses {0}. Which are
not whitelisted in LICENSE_FLAGS_WHITELIST".format(
+message = "because it has restricted licenses {0}. Which are
not whitelisted in LICENSE_FLAGS_ACCEPTED".format(
", ".join("'{0}'".format(f) for f in
unmatched_license_flags))
bb.debug(1, "Skipping %s %s" % (pn, message))
raise bb.parse.SkipRecipe(message)
diff --git a/meta/classes/license.bbclass b/meta/classes/license.bbclass
index d5480d87e24..dd1e07ee377 100644
--- a/meta/classes/license.bbclass
+++ b/meta/classes/license.bbclass
@@ -392,7 +392,7 @@ def check_license_flags(d):
license_flags = d.getVar('LICENSE_FLAGS')
if license_flags:
-whitelist = d.getVar('LICENSE_FLAGS_WHITELIST')
+whitelist = d.getVar('LICENSE_FLAGS_ACCEPTED')
if not whitelist:
return license_flags.split()
unmatched_flags = all_license_flags_match(license_flags, whitelist)
diff --git a/meta/conf/bitbake.conf b/meta/conf/bitbake.conf
index 3af649ce594..6fb7bfeb23c 100644
--- a/meta/conf/bitbake.conf
+++ b/meta/conf/bitbake.conf
@@ -108,6 +108,7 @@ BB_RENAMED_VARIABLES[ICECC_USER_PACKAGE_BL] =
"ICECC_RECIPE_DISABLE"
BB_RENAMED_VARIABLES[ICECC_SYSTEM_PACKAGE_BL] = "ICECC_RECIPE_DISABLE"
BB_RENAMED_VARIABLES[INHERIT_BLACKLIST] = "is a deprecated variable and no
longer needed"
BB_RENAMED_VARIABLES[TUNEABI_WHITELIST] = "is a deprecated variable and
support has been removed"
+BB_RENAMED_VARIABLES[LICENSE_FLAGS_WHITELIST] = "LICENSE_FLAGS_ACCEPTED"
##
# Architecture-dependent build variables.
diff --git a/meta/lib/oeqa/selftest/cases/distrodata.py
b/meta/lib/oeqa/selftest/cases/distrodata.py
index 908979804ab..03f31e9fcbf 100644
--- a/meta/lib/oeqa/selftest/cases/distrodata.py
+++ b/meta/lib/oeqa/selftest/cases/distrodata.py
@@ -18,7 +18,7 @@ class Distrodata(OESelftestTestCase):
Product: oe-core
Author: Alexander Kanavin
"""
-feature = 'LICENSE_FLAGS_WHITELIST += " commercial"\n'
+feature = 'LICENSE_FLAGS_ACCEPTED += " commercial"\n'
self.write_config(feature)
pkgs = oe.recipeutils.get_recipe_upgrade_status()
@@ -99,7 +99,7 @@ The following recipes do not have a DESCRIPTION. Please add
an entry for DESCRIP
return True
return False
-feature = 'require
conf/distro/include/maintainers.inc\nLICENSE_FLAGS_WHITELIST += "
commercial"\nPARSE_ALL_RECIPES = "1"\nPACKAGE_CLASSES = "package_ipk
package_deb package_rpm"\n'
+feature = 'require
conf/distro/include/maintainers.inc\nLICENSE_FLAGS_ACCEPTED += "
[OE-core] Proposal: INCOMPATIBLE_LICENSE_EXCEPTION
Folks, As a follow-on to yesterday's email and replies, I would like to make the following proposal for dealing with the changes to INCOMPATIBLE_LICENSE and associated variables. Current Usage: INCOMPATIBLE_LICENSE is a list of licenses that are considered incompatible with a distro's requirements. This is used to compare against packages built by a given recipe. A set of exception variables based on the license name (currently WHITELIST_) that contains a list of recipes that will be checked against the current recipe (PN) being evaluated. If it's in that list then all packages in that recipe will be built and included and the rest of the evaluation will be skipped. Otherwise, the packages (PKGS) from the recipe will be evaluated to see if any have a package specific license (LICENSE:). If a package has a license other than the INCOMPATIBLE_LICENSE the recipe will be built and any packages with the INCOMPATIBLE_LICENSE will be excluded from being packaged in package.bbclass via LICENSE_EXCLUSION- internal variable. The exception is predominately used for GPLv3 related packages, based on the emails replies overnight. Proposal: Keep the existing INCOMPATIBLE_LICENSE variable with the same behavior. The values in INCOMPATIBLE_LICENSE should be SDPX normalized license strings. As Richard has already suggested an alternative variable that is more meaningful: INCOMPATIBLE_LICENSE_EXCEPTION with an a : value. Rename the LICENSE_EXCLUSION- variable to make it clear that it is an internal variable. The usage of the _EXCEPTION variable should contain pkg names not recipe name. ** This would be an important change ** Clean up code as appropriate to ensure the exceptions are handled once and identified during parsing. I will start working on the implementation, Monday is a holiday in the US, so this should give sometime for this to be reviewed for Tuesday. -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161951): https://lists.openembedded.org/g/openembedded-core/message/161951 Mute This Topic: https://lists.openembedded.org/mt/89240704/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] INCOMPATIBLE_LICENSES and WHITELIST_ usage
Folks, I am working on a proposal to re-write how INCOMPATIBLE_LICENSES is used and processed to possibly include a COMPATIBLE_LICENSES variable as well, see PeterK's email [0] I am trying to determine the usage of WHITELIST_ which would be used to override a license that might be listed in INCOMPATIBLE_LICENSES variable. Randy and I have done a quick and dirty survey of a 100 or so layers (thanks Randy) and could not find any real usage other than what's currently in OE-Core for WHITELIST_GPL-3.0. If you are using WHITELIST_, please let me reply with your usage. [0] https://lists.openembedded.org/g/openembedded-devel/message/95166 -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161870): https://lists.openembedded.org/g/openembedded-core/message/161870 Mute This Topic: https://lists.openembedded.org/mt/89221731/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [master-next] convert-variable-renames: change f-string back
Add back Copyright info with update.
Signed-off-by: Saul Wold
---
scripts/contrib/convert-variable-renames.py | 11 +++
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/scripts/contrib/convert-variable-renames.py
b/scripts/contrib/convert-variable-renames.py
index 28a3df597e..bc3e01887c 100755
--- a/scripts/contrib/convert-variable-renames.py
+++ b/scripts/contrib/convert-variable-renames.py
@@ -3,6 +3,9 @@
# Conversion script to rename variables to versions with improved terminology.
# Also highlights potentially problematic langage and removed variables.
#
+# Copyright (C) 2021 Richard Purdie
+# Copyright (C) 2022 Wind River Systems, Inc.
+#
# SPDX-License-Identifier: GPL-2.0-only
#
@@ -56,7 +59,7 @@ context_check_list = [
def processfile(fn):
-print(f"processing file '{fn}'")
+print("processing file '%s'" % fn)
try:
fh, abs_path = tempfile.mkstemp()
modified = False
@@ -75,13 +78,13 @@ def processfile(fn):
# Find removed names
for removed_name in removed_list:
if removed_name in line:
-print(f"{fn} needs further work at line {lineno}
because {removed_name} has been deprecated")
+print("%s needs further work at line %s because
has been deprecated" % (fn, lineno, remove_name))
for check_word in context_check_list:
if re.search(check_word, line, re.IGNORECASE):
-print(f"{fn} needs further work at line {lineno}
since it contains {check_word}")
+print("%s needs further work at line %s since it
contains %s"% (fn, lineno, check_word))
new_file.write(line)
if modified:
-print(f"*** Modified file '{fn}'")
+print("*** Modified file '%s'" % (fn))
shutil.copymode(fn, abs_path)
os.remove(fn)
shutil.move(abs_path, fn)
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161848):
https://lists.openembedded.org/g/openembedded-core/message/161848
Mute This Topic: https://lists.openembedded.org/mt/89214903/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] convert-variables: Script for Inclusive Language variable renames
From: Saul Wold
This script searches for a list of variable that have been renamed
and converts them to their more descriptive names. It also searches
for a list of variables that have been removed or deprecated and
prints a message.
It will print a message to inform the user that there are terms that
need to be updated in their files. Many of these changes are context
sensitive and may not be modified as they might be existing calls to
other libraries. This message is informational only.
I have tested this on poky and meta-openembedded so far.
(From OE-Core rev: 50fe7ba8dba05a9681c9095506f798796cfc2750)
Signed-off-by: Saul Wold
---
v2: renamed script, removed bitbake internal vars, added WHITELIST_ option
scripts/contrib/convert-variables.py | 110 +++
1 file changed, 110 insertions(+)
create mode 100755 scripts/contrib/convert-variables.py
diff --git a/scripts/contrib/convert-variables.py
b/scripts/contrib/convert-variables.py
new file mode 100755
index 00..a632fd4d5c
--- /dev/null
+++ b/scripts/contrib/convert-variables.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+#
+# Conversion script to rename variables with more descriptive terms
+#
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import re
+import os
+import sys
+import tempfile
+import shutil
+import mimetypes
+
+if len(sys.argv) < 2:
+print("Please specify a directory to run the conversion script against.")
+sys.exit(1)
+
+renames = {
+"BB_ENV_WHITELIST":"BB_ENV_PASSTHROUGH",
+"BB_ENV_EXTRAWHITE":"BB_ENV_PASSTHROUGH_ADDITIONS",
+"BB_HASHCONFIG_WHITELIST":"BB_HASHCONFIG_IGNORE_VARS",
+"BB_SETSCENE_ENFORCE_WHITELIST":"BB_SETSCENE_ENFORCE_IGNORE_TASKS",
+"BB_HASHBASE_WHITELIST":"BB_BASEHASH_IGNORE_VARS",
+"BB_HASHTASK_WHITELIST":"BB_TASKHASH_IGNORE_TASKS",
+"CVE_CHECK_PN_WHITELIST":"CVE_CHECK_SKIP_RECIPE",
+"CVE_CHECK_WHITELIST":"CVE_CHECK_IGNORE",
+"MULTI_PROVIDER_WHITELIST":"BB_MULTI_PROVIDER_ALLOWED",
+"PNBLACKLIST":"SKIP_RECIPE",
+"SDK_LOCAL_CONF_BLACKLIST":"ESDK_LOCAL_CONF_ALLOW",
+"SDK_LOCAL_CONF_WHITELIST":"ESDK_LOCAL_CONF_REMOVE",
+"SDK_INHERIT_BLACKLIST":"ESDK_CLASS_INHERIT_DISABLE",
+"SSTATE_DUPWHITELIST":"SSTATE_ALLOW_OVERLAP_FILES",
+"SYSROOT_DIRS_BLACKLIST":"SYSROOT_DIRS_IGNORE",
+"UNKNOWN_CONFIGURE_WHITELIST":"UNKNOWN_CONFIGURE_OPT_IGNORE",
+"WHITELIST_":"INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_",
+}
+
+removed_list = [
+"BB_STAMP_WHITELIST",
+"BB_STAMP_POLICY",
+"ICECC_USER_CLASS_BL",
+"ICECC_USER_PACKAGE_BL",
+"ICECC_USER_PACKAGE_WL",
+"INHERIT_BLACKLIST",
+"TUNEABI_WHITELIST",
+]
+
+context_check_list = [
+"blacklist",
+"whitelist",
+"abort",
+]
+
+def processfile(fn):
+
+print(f"processing file '{fn}'")
+try:
+fh, abs_path = tempfile.mkstemp()
+modified = False
+with os.fdopen(fh, 'w') as new_file:
+with open(fn, "r") as old_file:
+lineno = 0
+for line in old_file:
+lineno += 1
+if not line or "BB_RENAMED_VARIABLE" in line:
+continue
+# Do the renames
+for old_name, new_name in renames.items():
+if old_name in line:
+line = line.replace(old_name, new_name)
+modified = True
+# Find removed names
+for removed_name in removed_list:
+if removed_name in line:
+print(f"{fn} needs further work at line {lineno}
because {removed_name} has been deprecated")
+for check_word in context_check_list:
+if re.search(check_word, line, re.IGNORECASE):
+print(f"{fn} needs further work at line {lineno}
since it contains {check_word}")
+new_file.write(line)
+if modified:
+print(f"*** Modified file '{fn}'")
+shutil.copymode(fn, abs_path)
+os.remove(fn)
+shutil.move(abs_path, fn)
+except UnicodeDecodeError:
+pass
+
+ourname = os.path.basename(sys.argv[0])
+ourversion = "0.1"
+
+if os.path.isfile(sys.argv[1]):
+processfile(sys.argv[1])
+sys.exit(0)
+
+for targetdir in sys.argv[1:]:
+print("processing directory '%s'" % targetdir)
+for root, dirs, files in os.walk(targetdir):
+for name in files:
+if name == ourname:
+
[OE-core] [RFC PATCH] Rename INCOMPATIBLE related varibale
This is one option to handle the rename, another alternative would
be to change the new INCOMPATIBLE_LICENSE_ALLOWED_RECIPES to VarFlag
style.
Signed-off-by: Saul Wold
---
meta/classes/base.bbclass| 4 ++--
meta/classes/license_image.bbclass | 2 +-
meta/classes/multilib.bbclass| 6 +++---
meta/conf/distro/include/default-distrovars.inc | 2 +-
meta/lib/oeqa/selftest/cases/incompatible_lic.py | 2 +-
5 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 87a4cb5fc7..a376dfa134 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -598,9 +598,9 @@ python () {
whitelist = []
for lic in bad_licenses:
spdx_license = return_spdx(d, lic)
-whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
+
whitelist.extend((d.getVar("INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_" + lic) or
"").split())
if spdx_license:
-whitelist.extend((d.getVar("WHITELIST_" + spdx_license) or
"").split())
+
whitelist.extend((d.getVar("INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_" +
spdx_license) or "").split())
if pn in whitelist:
'''
diff --git a/meta/classes/license_image.bbclass
b/meta/classes/license_image.bbclass
index bf70bee99b..701516d324 100644
--- a/meta/classes/license_image.bbclass
+++ b/meta/classes/license_image.bbclass
@@ -56,7 +56,7 @@ def write_license_files(d, license_manifest, pkg_dic,
rootfs=True):
whitelist = []
for lic in bad_licenses:
-whitelist.extend((d.getVar("WHITELIST_" + lic) or "").split())
+whitelist.extend((d.getVar("INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_" +
lic) or "").split())
with open(license_manifest, "w") as license_file:
for pkg in sorted(pkg_dic):
diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass
index ec2013198c..278a88181a 100644
--- a/meta/classes/multilib.bbclass
+++ b/meta/classes/multilib.bbclass
@@ -75,11 +75,11 @@ python multilib_virtclass_handler () {
e.data.setVar("PN", variant + "-" + e.data.getVar("PN", False))
e.data.setVar("OVERRIDES", e.data.getVar("OVERRIDES", False) + override)
-# Expand WHITELIST_GPL-3.0 with multilib prefix
-pkgs = e.data.getVar("WHITELIST_GPL-3.0")
+# Expand INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_GPL-3.0 with multilib prefix
+pkgs = e.data.getVar("INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_GPL-3.0")
for pkg in pkgs.split():
pkgs += " " + variant + "-" + pkg
-e.data.setVar("WHITELIST_GPL-3.0", pkgs)
+e.data.setVar("INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_GPL-3.0", pkgs)
# DEFAULTTUNE can change TARGET_ARCH override so expand this now before
update_data
newtune = e.data.getVar("DEFAULTTUNE:" + "virtclass-multilib-" + variant,
False)
diff --git a/meta/conf/distro/include/default-distrovars.inc
b/meta/conf/distro/include/default-distrovars.inc
index 3bba651a77..7bfb946e67 100644
--- a/meta/conf/distro/include/default-distrovars.inc
+++ b/meta/conf/distro/include/default-distrovars.inc
@@ -20,7 +20,7 @@ DISTRO_FEATURES_DEFAULT ?= "acl alsa argp bluetooth
debuginfod ext2 ipv4 ipv6 la
DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT}"
IMAGE_FEATURES ?= ""
-WHITELIST_GPL-3.0 ?= ""
+INCOMPATIBLE_LICENSE_ALLOWED_RECIPE_GPL-3.0 ?= ""
COMMERCIAL_AUDIO_PLUGINS ?= ""
# COMMERCIAL_AUDIO_PLUGINS ?= "gst-plugins-ugly-mad
gst-plugins-ugly-mpegaudioparse"
diff --git a/meta/lib/oeqa/selftest/cases/incompatible_lic.py
b/meta/lib/oeqa/selftest/cases/incompatible_lic.py
index fd3b3f409e..52c381dba6 100644
--- a/meta/lib/oeqa/selftest/cases/incompatible_lic.py
+++ b/meta/lib/oeqa/selftest/cases/incompatible_lic.py
@@ -111,7 +111,7 @@ INCOMPATIBLE_LICENSE:pn-core-image-minimal = "GPL-3.0
LGPL-3.0"
bitbake('core-image-minimal')
def test_bash_whitelist(self):
-self.write_config(self.default_config() +
'\nWHITELIST_GPL-3.0:pn-core-image-minimal = "bash"')
+self.write_config(self.default_config() +
'\nINCOMPATIBLE_LICENSE_ALLOWED_RECIPE_GPL-3.0:pn-core-image-minimal = "bash"')
bitbake('core-image-minimal')
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161790):
https://lists.openembedded.org/g/openembedded-core/message/161790
Mute This Topic: https://lists.openembedded.org/mt/89198826/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [WIP] descriptive_rename: Script for Inclusive Language variable renames
This script searches for a list of variable that have been renamed
and converts them to their more descriptive names. It also searches
for a list of variables that have been removed or deprecated and
prints a message.
It will print a message to inform the user that there are terms that
need to be updated in their files. Many of these changes are context
sensitive and may not be modified as they might be existing calls to
other libraries. This message is informational only.
I have tested this on poky and meta-openembedded so far.
Signed-off-by: Saul Wold
---
scripts/contrib/descriptive-rename.py | 113 ++
1 file changed, 113 insertions(+)
create mode 100755 scripts/contrib/descriptive-rename.py
diff --git a/scripts/contrib/descriptive-rename.py
b/scripts/contrib/descriptive-rename.py
new file mode 100755
index 00..f036328647
--- /dev/null
+++ b/scripts/contrib/descriptive-rename.py
@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+#
+# Conversion script to rename variables with more descriptive terms
+#
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import re
+import os
+import sys
+import tempfile
+import shutil
+import mimetypes
+
+if len(sys.argv) < 2:
+print("Please specify a directory to run the conversion script against.")
+sys.exit(1)
+
+
+renames = {
+"BB_ENV_WHITELIST":"BB_ENV_PASSTHROUGH",
+"BB_ENV_EXTRAWHITE":"BB_ENV_PASSTHROUGH_ADDITIONS",
+"BB_HASHCONFIG_WHITELIST":"BB_HASHCONFIG_IGNORE_VARS",
+"BB_SETSCENE_ENFORCE_WHITELIST":"BB_SETSCENE_ENFORCE_IGNORE_TASKS",
+"BB_HASHBASE_WHITELIST":"BB_BASEHASH_IGNORE_VARS",
+"BB_HASHTASK_WHITELIST":"BB_TASKHASH_IGNORE_TASKS",
+"CVE_CHECK_PN_WHITELIST":"CVE_CHECK_SKIP_RECIPE",
+"CVE_CHECK_WHITELIST":"CVE_CHECK_IGNORE",
+"MULTI_PROVIDER_WHITELIST":"BB_MULTI_PROVIDER_ALLOWED",
+"PNBLACKLIST":"SKIP_RECIPE",
+"SDK_LOCAL_CONF_BLACKLIST":"ESDK_LOCAL_CONF_ALLOW",
+"SDK_LOCAL_CONF_WHITELIST":"ESDK_LOCAL_CONF_REMOVE",
+"SDK_INHERIT_BLACKLIST":"ESDK_CLASS_INHERIT_DISABLE",
+"SSTATE_DUPWHITELIST":"SSTATE_ALLOW_OVERLAP_FILES",
+"SYSROOT_DIRS_BLACKLIST":"SYSROOT_DIRS_IGNORE",
+"UNKNOWN_CONFIGURE_WHITELIST":"UNKNOWN_CONFIGURE_OPT_IGNORE",
+"basewhitelist":"basehash_ignore_vars",
+"taskwhitelist":"taskhash_ignore_tasks",
+}
+
+removed_list = [
+"BB_STAMP_WHITELIST",
+"BB_STAMP_POLICY",
+"ICECC_USER_CLASS_BL",
+"ICECC_USER_PACKAGE_BL",
+"ICECC_USER_PACKAGE_WL",
+"INHERIT_BLACKLIST",
+"WHITELIST_",
+"TUNEABI_WHITELIST",
+]
+
+context_check_list = [
+"blacklist",
+"whitelist",
+"abort",
+]
+
+def processfile(fn):
+
+print(f"processing file '{fn}'")
+try:
+fh, abs_path = tempfile.mkstemp()
+modified = False
+with os.fdopen(fh, 'w') as new_file:
+with open(fn, "r") as old_file:
+lineno = 0
+for line in old_file:
+lineno += 1
+if not line or "BB_RENAMED_VARIABLE" in line:
+continue
+# Do the renames
+for old_name, new_name in renames.items():
+if old_name in line:
+line = line.replace(old_name, new_name)
+modified = True
+# Find removed names
+for removed_name in removed_list:
+if removed_name in line:
+print(f"{fn} needs further work at line {lineno}
because {removed_name} has been deprecated")
+for check_word in context_check_list:
+if re.search(check_word, line, re.IGNORECASE):
+print(f"{fn} needs further work at line {lineno}
since it contains {check_word}")
+new_file.write(line)
+if modified:
+print(f"*** Modified file '{fn}'")
+shutil.copymode(fn, abs_path)
+os.remove(fn)
+shutil.move(abs_path, fn)
+except UnicodeDecodeError:
+pass
+
+ourname = os.path.basename(sys.argv[0])
+ourversion = "0.1"
+
+if os.path.isfile(sys.argv[1]):
+processfile(sys.argv[1])
+sys.exit(0)
+
+for targetdir in sys.argv[1:]:
+print("processing directory '%s'" % targetdir)
+for root, dirs, files in os.walk(targetdir):
+for name in files:
+if name == ourname:
+continue
+fn = os.path.join(root, name)
+i
Re: [OE-core] [PATCH v3] create-spdx: Get SPDX-License-Identifier from source
On 2/8/22 07:37, Konrad Weihmann wrote:
On 08.02.22 16:02, Saul Wold wrote:
This patch will read the begining of source files and try to find
the SPDX-License-Identifier to populate the licenseInfoInFiles
field for each source file. This does not populate licenseConcluded
at this time, nor rolls it up to package level.
We read as binary file since some source code seem to have some
binary characters, the license is then converted to ascii strings.
Signed-off-by: Saul Wold
---
v2: Clean up commit message
v3: Really fix up regex based on Peter's feedback!
meta/classes/create-spdx.bbclass | 22 ++
1 file changed, 22 insertions(+)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 8b4203fdb5..64aada8593 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -37,6 +37,23 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for
SPDX packages created f
do_image_complete[depends] = "virtual/kernel:do_create_spdx"
+def extract_licenses(filename):
+import re
+
+lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([
\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
Taking inspiration from reuse-tool
(https://github.com/fsfe/reuse-tool/blob/master/src/reuse/_comment.py)
and the way they parse comment blocks the results with the updated regex
look good.
I was not aware of this parser.
Test sample set:
(* SPDX-License-Identifier: Foo-Bar *)
(* SPDX-License-Identifier: Foo-Bar *)
/* SPDX-License-Identifier: Foo-Bar */
<#-- SPDX-License-Identifier: Foo-Bar -->
<%-- SPDX-License-Identifier: Foo-Bar --%>
{# SPDX-License-Identifier: Foo-Bar #}
{/* SPDX-License-Identifier: Foo-Bar */}
{{!-- SPDX-License-Identifier: Foo-Bar --}}
@Comment{ SPDX-License-Identifier: Foo-Bar } ---> Only this one is
missed (which is bibtex syntax) - no idea if that is of importance for
anyone.
Just wanted to highlight that this is not catching every possible input line
Do we need to pull in the complexity of the reuse-tool comment parser?
Let me know, might not make 3.5 if this is the case.
Sau!
+
+try:
+with open(filename, 'rb') as f:
+size = min(15000, os.stat(filename).st_size)
+txt = f.read(size)
+licenses = re.findall(lic_regex, txt)
+if licenses:
+ascii_licenses = [lic.decode('ascii') for lic in licenses]
+return ascii_licenses
+except Exception as e:
+bb.warn(f"Exception reading {filename}: {e}")
+return None
+
def get_doc_namespace(d, doc):
import uuid
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +249,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid, get_types, *, archiv
checksumValue=bb.utils.sha256_file(filepath),
))
+if "SOURCE" in spdx_file.fileTypes:
+extracted_lics = extract_licenses(filepath)
+if extracted_lics:
+spdx_file.licenseInfoInFiles = extracted_lics
+
doc.files.append(spdx_file)
doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161517):
https://lists.openembedded.org/g/openembedded-core/message/161517
Mute This Topic: https://lists.openembedded.org/mt/88997967/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v3] create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find
the SPDX-License-Identifier to populate the licenseInfoInFiles
field for each source file. This does not populate licenseConcluded
at this time, nor rolls it up to package level.
We read as binary file since some source code seem to have some
binary characters, the license is then converted to ascii strings.
Signed-off-by: Saul Wold
---
v2: Clean up commit message
v3: Really fix up regex based on Peter's feedback!
meta/classes/create-spdx.bbclass | 22 ++
1 file changed, 22 insertions(+)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 8b4203fdb5..64aada8593 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -37,6 +37,23 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for
SPDX packages created f
do_image_complete[depends] = "virtual/kernel:do_create_spdx"
+def extract_licenses(filename):
+import re
+
+lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([
\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
+
+try:
+with open(filename, 'rb') as f:
+size = min(15000, os.stat(filename).st_size)
+txt = f.read(size)
+licenses = re.findall(lic_regex, txt)
+if licenses:
+ascii_licenses = [lic.decode('ascii') for lic in licenses]
+return ascii_licenses
+except Exception as e:
+bb.warn(f"Exception reading {filename}: {e}")
+return None
+
def get_doc_namespace(d, doc):
import uuid
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +249,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid, get_types, *, archiv
checksumValue=bb.utils.sha256_file(filepath),
))
+if "SOURCE" in spdx_file.fileTypes:
+extracted_lics = extract_licenses(filepath)
+if extracted_lics:
+spdx_file.licenseInfoInFiles = extracted_lics
+
doc.files.append(spdx_file)
doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161513):
https://lists.openembedded.org/g/openembedded-core/message/161513
Mute This Topic: https://lists.openembedded.org/mt/88997967/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find
the SPDX-License-Identifier to populate the licenseInfoInFiles
field for each source file. This does not populate licenseConcluded
at this time, nor rolls it up to package level.
We read as binary file since some source code seem to have some
binary characters, the license is then converted to ascii strings.
Signed-off-by: Saul Wold
---
v2: Updated commit message, and fixed REGEX based on Peter's suggetion
meta/classes/create-spdx.bbclass | 23 +++
1 file changed, 23 insertions(+)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 8b4203fdb5..588489cc2b 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for
SPDX packages created f
do_image_complete[depends] = "virtual/kernel:do_create_spdx"
+def extract_licenses(filename):
+import re
+import oe.spdx
+
+lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[
|\n|\r\n]*?')
+
+try:
+with open(filename, 'rb') as f:
+size = min(15000, os.stat(filename).st_size)
+txt = f.read(size)
+licenses = re.findall(lic_regex, txt)
+if licenses:
+ascii_licenses = [lic.decode('ascii') for lic in licenses]
+return ascii_licenses
+except Exception as e:
+bb.warn(f"Exception reading {filename}: {e}")
+return None
+
def get_doc_namespace(d, doc):
import uuid
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid, get_types, *, archiv
checksumValue=bb.utils.sha256_file(filepath),
))
+if "SOURCE" in spdx_file.fileTypes:
+extracted_lics = extract_licenses(filepath)
+if extracted_lics:
+spdx_file.licenseInfoInFiles = extracted_lics
+
doc.files.append(spdx_file)
doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161465):
https://lists.openembedded.org/g/openembedded-core/message/161465
Mute This Topic: https://lists.openembedded.org/mt/88980079/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/5] imagefeatures: selftest: Change variable to be more descriptive
This changes a couple of variables to be more representive of
their usage.
Signed-off-by: Saul Wold
---
meta/lib/oeqa/selftest/cases/imagefeatures.py | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/meta/lib/oeqa/selftest/cases/imagefeatures.py
b/meta/lib/oeqa/selftest/cases/imagefeatures.py
index 18f37c6d7d9..d36d45c5516 100644
--- a/meta/lib/oeqa/selftest/cases/imagefeatures.py
+++ b/meta/lib/oeqa/selftest/cases/imagefeatures.py
@@ -198,8 +198,8 @@ class ImageFeatures(OESelftestTestCase):
image_name = 'core-image-minimal'
all_image_types = set(get_bb_var("IMAGE_TYPES", image_name).split())
-blacklist = set(('container', 'elf', 'f2fs', 'multiubi', 'tar.zst',
'wic.zst'))
-img_types = all_image_types - blacklist
+skip_image_types = set(('container', 'elf', 'f2fs', 'multiubi',
'tar.zst', 'wic.zst'))
+img_types = all_image_types - skip_image_types
config = 'IMAGE_FSTYPES += "%s"\n'\
'MKUBIFS_ARGS ?= "-m 2048 -e 129024 -c 2047"\n'\
@@ -245,8 +245,8 @@ VIRTUAL-RUNTIME_base-utils = "packagegroup-core-base-utils"
VIRTUAL-RUNTIME_base-utils-hwclock = "util-linux-hwclock"
VIRTUAL-RUNTIME_base-utils-syslog = ""
-# Blacklist busybox
-PNBLACKLIST[busybox] = "Don't build this"
+# Skip busybox
+SKIP_RECIPE[busybox] = "Don't build this"
"""
self.write_config(config)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161372):
https://lists.openembedded.org/g/openembedded-core/message/161372
Mute This Topic: https://lists.openembedded.org/mt/88911228/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/5] Use more descriptive variable for skipping recipes
This patchset in conjuction with a meta-yocto and documentation patch change the varFlag for skipping a recipe to be SKIP_RECIPE. This removes the need to have to explicitly use the class by moving the code to base.bbclass. Tested with the Autobuilder (a-quick) and by using the PNBLACKLIST variable. Sau! Saul Wold (5): skip_recipe: remove old class and rename VarFlag to SKIP_RECIPE imagefeatures: selftest: Change variable to be more descriptive multilib: Use renamed SKIP_RECIPE varFlag dnf: Use renamed SKIP_RECIPE varFlag documentation: Update for skip_recipe rename meta/classes/base.bbclass | 10 ++ meta/classes/blacklist.bbclass| 20 --- meta/classes/multilib.bbclass | 8 meta/conf/distro/defaultsetup.conf| 3 +-- meta/conf/documentation.conf | 2 +- meta/lib/oeqa/selftest/cases/imagefeatures.py | 8 meta/recipes-devtools/dnf/dnf_4.10.0.bb | 2 +- meta/recipes-devtools/libdnf/libdnf_0.65.0.bb | 2 +- 8 files changed, 22 insertions(+), 33 deletions(-) delete mode 100644 meta/classes/blacklist.bbclass -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161373): https://lists.openembedded.org/g/openembedded-core/message/161373 Mute This Topic: https://lists.openembedded.org/mt/88911229/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/5] skip_recipe: remove old class and rename VarFlag to SKIP_RECIPE
This change better describes what the VarFlag is doing since it
is implemeted with the SkipRecipe() function.
By moving this into base.bbclass we simplify the distro inherit.
Signed-off-by: Saul Wold
---
meta/classes/base.bbclass | 10 ++
meta/classes/blacklist.bbclass | 20
meta/conf/distro/defaultsetup.conf | 3 +--
3 files changed, 11 insertions(+), 22 deletions(-)
delete mode 100644 meta/classes/blacklist.bbclass
diff --git a/meta/classes/base.bbclass b/meta/classes/base.bbclass
index 5f4956a1d31..854d14d8a51 100644
--- a/meta/classes/base.bbclass
+++ b/meta/classes/base.bbclass
@@ -438,6 +438,16 @@ python () {
if os.path.normpath(d.getVar("WORKDIR")) !=
os.path.normpath(d.getVar("B")):
d.appendVar("PSEUDO_IGNORE_PATHS", ",${B}")
+# To add a recipe to the skip list , set:
+# SKIP_RECIPE[pn] = "message"
+pn = d.getVar('PN')
+if d.getVarFlag('PNBLACKLIST', pn) is not None:
+bb.error("PNBLACKLIST is deprecated, please convert to
SKIP_RECIPE[%s]" % (pn))
+skip_msg = d.getVarFlag('SKIP_RECIPE', pn)
+if skip_msg:
+bb.debug(1, "Skipping %s %s" % (pn, skip_msg))
+raise bb.parse.SkipRecipe("Recipe will be skipped because: %s" %
(skip_msg))
+
# Handle PACKAGECONFIG
#
# These take the form:
diff --git a/meta/classes/blacklist.bbclass b/meta/classes/blacklist.bbclass
deleted file mode 100644
index dc794228ffe..000
--- a/meta/classes/blacklist.bbclass
+++ /dev/null
@@ -1,20 +0,0 @@
-# anonymous support class from originally from angstrom
-#
-# To use the blacklist, a distribution should include this
-# class in the INHERIT_DISTRO
-#
-# No longer use ANGSTROM_BLACKLIST, instead use a table of
-# recipes in PNBLACKLIST
-#
-# Features:
-#
-# * To add a package to the blacklist, set:
-# PNBLACKLIST[pn] = "message"
-#
-
-python () {
-blacklist = d.getVarFlag('PNBLACKLIST', d.getVar('PN'))
-
-if blacklist:
-raise bb.parse.SkipRecipe("Recipe is blacklisted: %s" % (blacklist))
-}
diff --git a/meta/conf/distro/defaultsetup.conf
b/meta/conf/distro/defaultsetup.conf
index b36a4e5..f6894f3ab56 100644
--- a/meta/conf/distro/defaultsetup.conf
+++ b/meta/conf/distro/defaultsetup.conf
@@ -14,9 +14,8 @@ TMPDIR .= "${TCLIBCAPPEND}"
USER_CLASSES ?= ""
PACKAGE_CLASSES ?= "package_ipk"
-INHERIT_BLACKLIST = "blacklist"
INHERIT_DISTRO ?= "debian devshell sstate license remove-libtool"
-INHERIT += "${PACKAGE_CLASSES} ${USER_CLASSES} ${INHERIT_DISTRO}
${INHERIT_BLACKLIST}"
+INHERIT += "${PACKAGE_CLASSES} ${USER_CLASSES} ${INHERIT_DISTRO}"
INIT_MANAGER ??= "none"
require conf/distro/include/init-manager-${INIT_MANAGER}.inc
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161371):
https://lists.openembedded.org/g/openembedded-core/message/161371
Mute This Topic: https://lists.openembedded.org/mt/88911227/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 5/5] documentation: Update for skip_recipe rename
This change better aligns the name of the variable with it's purpose. Since we removed the odler class, the associated documentation is also removed. Signed-off-by: Saul Wold --- meta/conf/documentation.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/conf/documentation.conf b/meta/conf/documentation.conf index 6b50ad08a8b..9614448a1fe 100644 --- a/meta/conf/documentation.conf +++ b/meta/conf/documentation.conf @@ -331,7 +331,6 @@ PKGDATA_DIR[doc] = "Points to a shared, global-state directory that holds data g PKGDEST[doc] = "Points to the parent directory for files to be packaged after they have been split into individual packages." PKGDESTWORK[doc] = "Points to a temporary work area used by the do_package task to write output from the do_packagedata task." PN[doc] = "PN refers to a recipe name in the context of a file used by the OpenEmbedded build system as input to create a package. It refers to a package name in the context of a file created or produced by the OpenEmbedded build system." -PNBLACKLIST[doc] = "Lists recipes you do not want the OpenEmbedded build system to build." PR[doc] = "The revision of the recipe. The default value for this variable is 'r0'." PREFERRED_PROVIDER[doc] = "If multiple recipes provide an item, this variable determines which recipe should be given preference." PREFERRED_VERSION[doc] = "If there are multiple versions of recipes available, this variable determines which recipe should be given preference." @@ -385,6 +384,7 @@ SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS[doc] = "A list of recipe dependencies that shoul SIGGEN_EXCLUDERECIPES_ABISAFE[doc] = "A list of recipes that are completely stable and will never change." SITEINFO_BITS[doc] = "Specifies the number of bits for the target system CPU." SITEINFO_ENDIANNESS[doc] = "Specifies the endian byte order of the target system. The value should be either 'le' for 'little-endian' or 'be' for 'big-endian'." +SKIP_RECIPE[doc] = "Lists recipes you do not want the OpenEmbedded build system to build." SOC_FAMILY[doc] = "Groups together machines based upon the same family of SOC (System On Chip). You typically set this variable in a common .inc file that you include in the configuration files of all the machines." SOLIBS[doc] = "Defines the suffix for shared libraries used on the target platform." SOLIBSDEV[doc] = "Defines the suffix for the development symbolic link (symlink) for shared libraries on the target platform." -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#161370): https://lists.openembedded.org/g/openembedded-core/message/161370 Mute This Topic: https://lists.openembedded.org/mt/88911226/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 4/5] dnf: Use renamed SKIP_RECIPE varFlag
This is a more descriptive variable name updated in base.bbclass
Signed-off-by: Saul Wold
---
meta/recipes-devtools/dnf/dnf_4.10.0.bb | 2 +-
meta/recipes-devtools/libdnf/libdnf_0.65.0.bb | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/meta/recipes-devtools/dnf/dnf_4.10.0.bb
b/meta/recipes-devtools/dnf/dnf_4.10.0.bb
index 3fc24b132df..3f22c1d8f2e 100644
--- a/meta/recipes-devtools/dnf/dnf_4.10.0.bb
+++ b/meta/recipes-devtools/dnf/dnf_4.10.0.bb
@@ -87,4 +87,4 @@ SYSTEMD_SERVICE:${PN} = "dnf-makecache.service
dnf-makecache.timer \
"
SYSTEMD_AUTO_ENABLE ?= "disable"
-PNBLACKLIST[dnf] ?= "${@bb.utils.contains('PACKAGE_CLASSES', 'package_rpm',
'', 'does not build without package_rpm in PACKAGE_CLASSES due disabled rpm
support in libsolv', d)}"
+SKIP_RECIPE[dnf] ?= "${@bb.utils.contains('PACKAGE_CLASSES', 'package_rpm',
'', 'does not build without package_rpm in PACKAGE_CLASSES due disabled rpm
support in libsolv', d)}"
diff --git a/meta/recipes-devtools/libdnf/libdnf_0.65.0.bb
b/meta/recipes-devtools/libdnf/libdnf_0.65.0.bb
index 81da04c9ca1..092ad98d4b7 100644
--- a/meta/recipes-devtools/libdnf/libdnf_0.65.0.bb
+++ b/meta/recipes-devtools/libdnf/libdnf_0.65.0.bb
@@ -34,5 +34,5 @@ EXTRA_OECMAKE:append:class-native = " -DWITH_GIR=OFF"
EXTRA_OECMAKE:append:class-nativesdk = " -DWITH_GIR=OFF"
BBCLASSEXTEND = "native nativesdk"
-PNBLACKLIST[libdnf] ?= "${@bb.utils.contains('PACKAGE_CLASSES', 'package_rpm',
'', 'Does not build without package_rpm in PACKAGE_CLASSES due disabled rpm
support in libsolv', d)}"
+SKIP_RECIPE[libdnf] ?= "${@bb.utils.contains('PACKAGE_CLASSES', 'package_rpm',
'', 'Does not build without package_rpm in PACKAGE_CLASSES due disabled rpm
support in libsolv', d)}"
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161368):
https://lists.openembedded.org/g/openembedded-core/message/161368
Mute This Topic: https://lists.openembedded.org/mt/88911221/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/5] multilib: Use renamed SKIP_RECIPE varFlag
This is a more descriptive variable name updated in base.bbclass
Signed-off-by: Saul Wold
---
meta/classes/multilib.bbclass | 8
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/meta/classes/multilib.bbclass b/meta/classes/multilib.bbclass
index 4a3e582816d..ec2013198ce 100644
--- a/meta/classes/multilib.bbclass
+++ b/meta/classes/multilib.bbclass
@@ -65,11 +65,11 @@ python multilib_virtclass_handler () {
override = ":virtclass-multilib-" + variant
-blacklist = e.data.getVarFlag('PNBLACKLIST', e.data.getVar('PN'))
-if blacklist:
+skip_msg = e.data.getVarFlag('SKIP_RECIPE', e.data.getVar('PN'))
+if skip_msg:
pn_new = variant + "-" + e.data.getVar('PN')
-if not e.data.getVarFlag('PNBLACKLIST', pn_new):
-e.data.setVarFlag('PNBLACKLIST', pn_new, blacklist)
+if not e.data.getVarFlag('SKIP_RECIPE', pn_new):
+e.data.setVarFlag('SKIP_RECIPE', pn_new, skip_msg)
e.data.setVar("MLPREFIX", variant + "-")
e.data.setVar("PN", variant + "-" + e.data.getVar("PN", False))
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161369):
https://lists.openembedded.org/g/openembedded-core/message/161369
Mute This Topic: https://lists.openembedded.org/mt/88911225/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] recipetool/create: Scan for SDPX-License-Identifier
On 2/3/22 13:24, Richard Purdie wrote:
On Thu, 2022-02-03 at 09:07 -0800, Saul Wold wrote:
When a file can not be identified by checksum and they contain an SPDX
License-Identifier tag, use it as the found license.
[YOCTO #14529]
Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags
Signed-off-by: Saul Wold
---
scripts/lib/recipetool/create.py | 16 +++-
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 507a230511..9149c2d94f 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
for licfile in sorted(licfiles):
md5value = bb.utils.md5_file(licfile)
license = md5sums.get(md5value, None)
+license_list = []
if not license:
license, crunched_md5, lictext = crunch_license(licfile)
if lictext and not license:
-license = 'Unknown'
-logger.info("Please add the following line for '%s' to a
'lib/recipetool/licenses.csv' " \
-"and replace `Unknown` with the license:\n" \
-"%s,Unknown" % (os.path.relpath(licfile, srctree),
md5value))
-if license:
+spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d.
]+)[ |\n|\r\n]*?')
+license_list = re.findall(spdx_re, "\n".join(lictext))
+if not license_list:
+license_list.append('Unknown')
+logger.info("Please add the following line for '%s' to a
'lib/recipetool/licenses.csv' " \
+"and replace `Unknown` with the license:\n" \
+"%s,Unknown" % (os.path.relpath(licfile, srctree),
md5value))
+else:
+license_list.append(license)
+for license in license_list:
licenses.append((license, os.path.relpath(licfile, srctree),
md5value))
# FIXME should we grab at least one source file with a license header and add that too?
I think to close this bug the code may need to go one step further and
effectively grep over the source tree.
We'd probably want to list the value of any SPDX-License-Identifier: header
found in any of the source files for the user to then decide upon?
That's moving in to the create-spdx.bbclass territory I think. The
change would need to be much larger. and I will likely have to shelve
for a while.
Or am I misunderstanding?
Maybe it's my misunderstanding, Tim has mentioned the LICENSE related
files in the bug report.
Sau!
Cheers,
Richard
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161322):
https://lists.openembedded.org/g/openembedded-core/message/161322
Mute This Topic: https://lists.openembedded.org/mt/7504/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] recipetool: Fix circular reference in SRC_URI
When creating a new recipe.bb file for a binary, don't use BP which
includes the version information, instead use BPN which is just the
name base Package Name.
Since PB is not specified, it takes the default:
PV = "1.0+git${SRCPV}"
But SRCPV is defined in terms of the SRC_URI, which leads to infinite
recursion (traceback below). Here are the pertinent variables which
cause the recursion:
SRC_URI = "git://github.com/lvc/abi-dumper;protocol=https;subdir=${BP}"
BP = "${BPN}-${PV}"
PV = "1.0+git${SRCPV}"
SRCPV = "${@bb.fetch2.get_srcrev(d)}"
def get_srcrev(d, method_name='sortable_revision'):
# ... trimmed
scms = []
fetcher = Fetch(d.getVar('SRC_URI').split(), d)
# ... trimmed
[YOCTO #14040]
Signed-off-by: Saul Wold
---
scripts/lib/recipetool/create.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 507a230511..4cf6a5a95c 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -435,7 +435,7 @@ def create_recipe(args):
if args.binary:
# Assume the archive contains the directory structure verbatim
# so we need to extract to a subdirectory
-fetchuri += ';subdir=${BP}'
+fetchuri += ';subdir=${BPN}'
srcuri = fetchuri
rev_re = re.compile(';rev=([^;]+)')
res = rev_re.search(srcuri)
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161294):
https://lists.openembedded.org/g/openembedded-core/message/161294
Mute This Topic: https://lists.openembedded.org/mt/88891162/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] recipetool/create: Scan for SDPX-License-Identifier
When a file can not be identified by checksum and they contain an SPDX
License-Identifier tag, use it as the found license.
[YOCTO #14529]
Tested with LICENSE files that contain 1 or more SPDX-License-Identifier tags
Signed-off-by: Saul Wold
---
scripts/lib/recipetool/create.py | 16 +++-
1 file changed, 11 insertions(+), 5 deletions(-)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 507a230511..9149c2d94f 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -1221,14 +1221,20 @@ def guess_license(srctree, d):
for licfile in sorted(licfiles):
md5value = bb.utils.md5_file(licfile)
license = md5sums.get(md5value, None)
+license_list = []
if not license:
license, crunched_md5, lictext = crunch_license(licfile)
if lictext and not license:
-license = 'Unknown'
-logger.info("Please add the following line for '%s' to a
'lib/recipetool/licenses.csv' " \
-"and replace `Unknown` with the license:\n" \
-"%s,Unknown" % (os.path.relpath(licfile, srctree),
md5value))
-if license:
+spdx_re = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d.
]+)[ |\n|\r\n]*?')
+license_list = re.findall(spdx_re, "\n".join(lictext))
+if not license_list:
+license_list.append('Unknown')
+logger.info("Please add the following line for '%s' to a
'lib/recipetool/licenses.csv' " \
+"and replace `Unknown` with the license:\n" \
+"%s,Unknown" % (os.path.relpath(licfile, srctree),
md5value))
+else:
+license_list.append(license)
+for license in license_list:
licenses.append((license, os.path.relpath(licfile, srctree),
md5value))
# FIXME should we grab at least one source file with a license header and
add that too?
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161291):
https://lists.openembedded.org/g/openembedded-core/message/161291
Mute This Topic: https://lists.openembedded.org/mt/7504/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source
On 2/1/22 19:21, Peter Kjellerstedt wrote:
-Original Message-
From: openembedded-core@lists.openembedded.org
On Behalf Of Saul Wold
Sent: den 2 februari 2022 01:02
To: openembedded-core@lists.openembedded.org; jpewhac...@gmail.com
Cc: Saul Wold
Subject: [OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find
the SPDX-License-Identifier to populate the licenseInfoInFiles
field for each source file. This does not populate licenseConculed
I assume that should be "licenseConcluded".
Well that depends on if "we" want to take some "ownership" of the
conclusion as the "preparer". How would we handle the case of 2
SPDX-License-Identifiers tags in a file, is it an "AND" or an "OR"?
Simple example.
The description of licenseConcluded is:
"License expression for licenseConcluded. The licensing that the
preparer of this SPDX document has concluded, based on the evidence,
actually applies to the package."
At somepoint we might be able to fill in that field, but for now I think
we leave it as NOASSERTION.
Sau!
at this time, nor rolls it up to package level.
We read as binary to since some source code seem to have some
to -> too
binary characters, the license is then converted to ascii strings.
Signed-off-by: Saul Wold
---
Merge after Joshua's patch (spdx: Add set helper for list properties)
merges
meta/classes/create-spdx.bbclass | 23 +++
1 file changed, 23 insertions(+)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 8b4203fdb5d..588489cc2b0 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for
SPDX packages created f
do_image_complete[depends] = "virtual/kernel:do_create_spdx"
+def extract_licenses(filename):
+import re
+import oe.spdx
You do not use oe.spdx in this function.
+
+lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[
|\n|\r\n]*?')
I assume you meant:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)(?:
|\n|\r\n)*?')
Not that it really matters though, as it will yield the same result as:
lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)')
However, neither of the expressions above will correctly match all the
SPDX-License-Identifier examples at https://spdx.dev/ids/#how.
Use this instead:
lic_regex = re.compile(b'^\W*SPDX-License-Identifier:\s*([
\w\d.()+-]+?)(?:\s+\W*)?$', re.MULTILINE)
+
+try:
+with open(filename, 'rb') as f:
+size = min(15000, os.stat(filename).st_size)
+txt = f.read(size)
+licenses = re.findall(lic_regex, txt)
+if licenses:
+ascii_licenses = [lic.decode('ascii') for lic in licenses]
+return ascii_licenses
+except Exception as e:
+bb.warn(f"Exception reading {filename}: {e}")
+return None
+
def get_doc_namespace(d, doc):
import uuid
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid, get_types, *, archiv
checksumValue=bb.utils.sha256_file(filepath),
))
+if "SOURCE" in spdx_file.fileTypes:
+extracted_lics = extract_licenses(filepath)
+if extracted_lics:
+spdx_file.licenseInfoInFiles = extracted_lics
+
doc.files.append(spdx_file)
doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
2.31.1
//Peter
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161177):
https://lists.openembedded.org/g/openembedded-core/message/161177
Mute This Topic: https://lists.openembedded.org/mt/88847186/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find
the SPDX-License-Identifier to populate the licenseInfoInFiles
field for each source file. This does not populate licenseConculed
at this time, nor rolls it up to package level.
We read as binary to since some source code seem to have some
binary characters, the license is then converted to ascii strings.
Signed-off-by: Saul Wold
---
Merge after Joshua's patch (spdx: Add set helper for list properties) merges
meta/classes/create-spdx.bbclass | 23 +++
1 file changed, 23 insertions(+)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 8b4203fdb5d..588489cc2b0 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -37,6 +37,24 @@ SPDX_SUPPLIER[doc] = "The SPDX PackageSupplier field for
SPDX packages created f
do_image_complete[depends] = "virtual/kernel:do_create_spdx"
+def extract_licenses(filename):
+import re
+import oe.spdx
+
+lic_regex = re.compile(b'SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[
|\n|\r\n]*?')
+
+try:
+with open(filename, 'rb') as f:
+size = min(15000, os.stat(filename).st_size)
+txt = f.read(size)
+licenses = re.findall(lic_regex, txt)
+if licenses:
+ascii_licenses = [lic.decode('ascii') for lic in licenses]
+return ascii_licenses
+except Exception as e:
+bb.warn(f"Exception reading {filename}: {e}")
+return None
+
def get_doc_namespace(d, doc):
import uuid
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +250,11 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid, get_types, *, archiv
checksumValue=bb.utils.sha256_file(filepath),
))
+if "SOURCE" in spdx_file.fileTypes:
+extracted_lics = extract_licenses(filepath)
+if extracted_lics:
+spdx_file.licenseInfoInFiles = extracted_lics
+
doc.files.append(spdx_file)
doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
spdx_pkg.hasFiles.append(spdx_file.SPDXID)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161170):
https://lists.openembedded.org/g/openembedded-core/message/161170
Mute This Topic: https://lists.openembedded.org/mt/88847186/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [WIP/RFC] create-spdx: Get SPDX-License-Identifier from source
This patch will read the begining of source files and try to find
the SPDX-License-Identifier to populate the licenseInfoInFiles
field for each source file. This does not populate licenseConculed
at this time, nor rolls it up to package level.
Signed-off-by: Saul Wold
---
classes/create-spdx.bbclass | 25 +
lib/oe/spdx.py | 2 +-
2 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/classes/create-spdx.bbclass b/classes/create-spdx.bbclass
index 180d667..9c11945 100644
--- a/classes/create-spdx.bbclass
+++ b/classes/create-spdx.bbclass
@@ -30,6 +30,21 @@ SPDX_LICENSES ??= "${COREBASE}/meta/files/spdx-licenses.json"
do_image_complete[depends] = "virtual/kernel:do_create_spdx"
+def extract_licenses(filename):
+import re
+lic_regex = re.compile('SPDX-License-Identifier:\s+([-A-Za-z\d. ]+)[
|\n|\r\n]*?')
+
+try:
+with open(filename, 'r') as f:
+size = min(15000, os.stat(filename).st_size)
+txt = f.read(size)
+licenses = re.findall(lic_regex, txt)
+if licenses:
+return licenses
+except Exception as e:
+bb.warn(f"Exception on {filename}: {e}")
+return None
+
def get_doc_namespace(d, doc):
import uuid
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
@@ -232,6 +247,16 @@ def add_package_files(d, doc, spdx_pkg, topdir,
get_spdxid, get_types, *, archiv
checksumValue=bb.utils.sha256_file(filepath),
))
+if "SOURCES" in spdx_file.fileTypes:
+licenses = extract_licenses(filepath)
+if licenses is not None:
+for lic in licenses:
+spdx_file.licenseInfoInFiles.append(lic.strip())
+else:
+spdx_file.licenseInfoInFiles.append("NOASSERTATION")
+else:
+spdx_file.licenseInfoInFiles.append("NOASSERTATION")
+
doc.files.append(spdx_file)
doc.add_relationship(spdx_pkg, "CONTAINS", spdx_file)
spdx_pkg.hasFiles.append(spdx_file.SPDXID)
diff --git a/lib/oe/spdx.py b/lib/oe/spdx.py
index 9e7ced5..71e7c1c 100644
--- a/lib/oe/spdx.py
+++ b/lib/oe/spdx.py
@@ -236,7 +236,7 @@ class SPDXFile(SPDXObject):
fileName = _String()
licenseConcluded = _String(default="NOASSERTION")
copyrightText = _String(default="NOASSERTION")
-licenseInfoInFiles = _StringList(default=["NOASSERTION"])
+licenseInfoInFiles = _StringList()
checksums = _ObjectList(SPDXChecksum)
fileTypes = _StringList()
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#161084):
https://lists.openembedded.org/g/openembedded-core/message/161084
Mute This Topic: https://lists.openembedded.org/mt/88756042/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] create-spdx: add support for SDKs
Overall I think this is going in the right direction, I need to review
it a little deeper and check the actual output.
I am not sure that you tested this against master as you use the old _
override syntax vs using a :.
See note below.
Sau!
On 1/12/22 11:40, Andres Beltran wrote:
Signed-off-by: Andres Beltran
---
meta/classes/create-spdx.bbclass | 95 +---
meta/lib/oe/sbom.py | 6 +-
2 files changed, 68 insertions(+), 33 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index e44a204a8fc..d0f987315ee 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -556,7 +556,7 @@ python do_create_spdx() {
oe.sbom.write_doc(d, package_doc, "packages")
}
# NOTE: depending on do_unpack is a hack that is necessary to get it's
dependencies for archive the source
-addtask do_create_spdx after do_package do_packagedata do_unpack before
do_build do_rm_work
+addtask do_create_spdx after do_package do_packagedata do_unpack before
do_populate_sdk do_build do_rm_work
SSTATETASKS += "do_create_spdx"
do_create_spdx[sstate-inputdirs] = "${SPDXDEPLOY}"
@@ -788,28 +788,77 @@ def spdx_get_src(d):
do_rootfs[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
ROOTFS_POSTUNINSTALL_COMMAND =+ "image_combine_spdx ; "
+
+do_populate_sdk[recrdeptask] += "do_create_spdx do_create_runtime_spdx"
+POPULATE_SDK_POST_HOST_COMMAND_append_task-populate-sdk = " sdk_host_combine_spdx;
"
+POPULATE_SDK_POST_TARGET_COMMAND_append_task-populate-sdk = "
sdk_target_combine_spdx; > +
You using the older _append syntax vs newer :append syntax in master
python image_combine_spdx() {
+import os
+import oe.sbom
+from pathlib import Path
+from oe.rootfs import image_list_installed_packages
+
+image_name = d.getVar("IMAGE_NAME")
+image_link_name = d.getVar("IMAGE_LINK_NAME")
+imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
+img_spdxid = oe.sbom.get_image_spdxid(image_name)
+packages = image_list_installed_packages(d)
+
+combine_spdx(d, image_name, imgdeploydir, img_spdxid, packages)
+
+if image_link_name:
+image_spdx_path = imgdeploydir / (image_name + ".spdx.json")
+image_spdx_link = imgdeploydir / (image_link_name + ".spdx.json")
+image_spdx_link.symlink_to(os.path.relpath(image_spdx_path,
image_spdx_link.parent))
+
+def make_image_link(target_path, suffix):
+if image_link_name:
+link = imgdeploydir / (image_link_name + suffix)
+link.symlink_to(os.path.relpath(target_path, link.parent))
+
+spdx_tar_path = imgdeploydir / (image_name + ".spdx.tar.zst")
+make_image_link(spdx_tar_path, ".spdx.tar.zst")
+spdx_index_path = imgdeploydir / (image_name + ".spdx.index.json")
+make_image_link(spdx_index_path, ".spdx.index.json")
+}
+
+python sdk_host_combine_spdx() {
+sdk_combine_spdx(d, "host")
+}
+
+python sdk_target_combine_spdx() {
+sdk_combine_spdx(d, "target")
+}
+
+def sdk_combine_spdx(d, sdk_type):
+import oe.sbom
+from pathlib import Path
+from oe.sdk import sdk_list_installed_packages
+
+sdk_name = d.getVar("SDK_NAME") + "-" + sdk_type
+sdk_deploydir = Path(d.getVar("SDKDEPLOYDIR"))
+sdk_spdxid = oe.sbom.get_sdk_spdxid(sdk_name)
+sdk_packages = sdk_list_installed_packages(d, sdk_type == "target")
+combine_spdx(d, sdk_name, sdk_deploydir, sdk_spdxid, sdk_packages)
+
+def combine_spdx(d, rootfs_name, rootfs_deploydir, rootfs_spdxid, packages):
import os
import oe.spdx
import oe.sbom
import io
import json
-from oe.rootfs import image_list_installed_packages
from datetime import timezone, datetime
from pathlib import Path
import tarfile
import bb.compress.zstd
creation_time = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-image_name = d.getVar("IMAGE_NAME")
-image_link_name = d.getVar("IMAGE_LINK_NAME")
-
deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
-imgdeploydir = Path(d.getVar("IMGDEPLOYDIR"))
source_date_epoch = d.getVar("SOURCE_DATE_EPOCH")
doc = oe.spdx.SPDXDocument()
-doc.name = image_name
+doc.name = rootfs_name
doc.documentNamespace = get_doc_namespace(d, doc)
doc.creationInfo.created = creation_time
doc.creationInfo.comment = "This document was created by analyzing the source
of the Yocto recipe during the build."
@@ -821,14 +870,12 @@ python image_combine_spdx() {
image = oe.spdx.SPDXPackage()
image.name = d.getVar("PN")
image.versionInfo = d.getVar("PV")
-image.SPDXID = oe.sbom.get_image_spdxid(image_name)
+image.SPDXID = rootfs_spdxid
doc.packages.append(image)
spdx_package = oe.spdx.SPDXPackage()
-packages = image_list_installed_packages(d)
-
for name in sorted(packages.keys()):
pkg_spdx_path =
[OE-core] [PATCH 2/3] package: Add support for kernel stripping
Extend runstrip() to accept additional argument to enable
sharing it with the kernel do_strip() so that
KERNEL_IMAGE_STRIP_EXTRA_SECTIONS can be passed.
Since is_elf() understands kernel modules there is no need to keep a
seperate list for kernmodules or hardcode the values to runstrip.
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 17 +
meta/lib/oe/package.py | 9 -
2 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 09cd376f4af..4927fb99ff1 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -390,10 +390,6 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir,
debugappend, debugsrcdir,
dvar = d.getVar('PKGD')
objcopy = d.getVar("OBJCOPY")
-# We ignore kernel modules, we don't generate debug info files.
-if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
-return (file, sources)
-
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
origmode = os.stat(file)[stat.ST_MODE]
@@ -1122,7 +1118,6 @@ python split_and_strip_files () {
#
elffiles = {}
symlinks = {}
-kernmods = []
staticlibs = []
inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir"))
@@ -1145,9 +1140,6 @@ python split_and_strip_files () {
if file in skipfiles:
continue
-if file.endswith(".ko") and file.find("/lib/modules/") != -1:
-kernmods.append(file)
-continue
if oe.package.is_static_lib(file):
staticlibs.append(file)
continue
@@ -1164,8 +1156,11 @@ python split_and_strip_files () {
if not s:
continue
# Check its an executable
-if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-or ((file.startswith(libdir) or
file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
+if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) \
+or (s[stat.ST_MODE] & stat.S_IXOTH) \
+or ((file.startswith(libdir) or
file.startswith(baselibdir)) \
+and (".so" in f or ".node" in f)) \
+or (f.startswith('vmlinux') or ".ko" in f):
if cpath.islink(file):
checkelflinks[file] = ltarget
@@ -1312,8 +1307,6 @@ python split_and_strip_files () {
elf_file = int(elffiles[file])
#bb.note("Strip %s" % file)
sfiles.append((file, elf_file, strip))
-for f in kernmods:
-sfiles.append((f, 16, strip))
if (d.getVar('PACKAGE_STRIP_STATIC') == '1' or
d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
for f in staticlibs:
sfiles.append((f, 16, strip))
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index dd700cbb0c9..7d387ee81d5 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -16,7 +16,11 @@ def runstrip(arg):
# 8 - shared library
# 16 - kernel module
-(file, elftype, strip) = arg
+if len(arg) == 3:
+(file, elftype, strip) = arg
+extra_strip_sections = ''
+else:
+(file, elftype, strip, extra_strip_sections) = arg
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
@@ -40,6 +44,9 @@ def runstrip(arg):
# shared or executable:
elif elftype & 8 or elftype & 4:
stripcmd.extend(["--remove-section=.comment",
"--remove-section=.note"])
+if extra_strip_sections != '':
+for section in extra_strip_sections.split():
+stripcmd.extend(["--remove-section=" + section])
stripcmd.append(file)
bb.debug(1, "runstrip: %s" % stripcmd)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160494):
https://lists.openembedded.org/g/openembedded-core/message/160494
Mute This Topic: https://lists.openembedded.org/mt/88377979/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/3] kernel.bbclass: use common strip()
Re-use the runstrip() code from oe.packaging, for the kernel
stripping process. Since runstrip() is python the kernel do_strip()
need to be converted to Python also. The stripped kernel image
will be used for deployment in do_deploy(). This will allow the
package.bbclass to split an unstripped kernel binary for the debug
information and extended packagedata. The extended package data is
used by create-spdx.
Signed-off-by: Saul Wold
---
meta/classes/kernel.bbclass | 44 -
1 file changed, 19 insertions(+), 25 deletions(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 7685c6921fa..473e28be477 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -700,30 +700,19 @@ do_kernel_link_images() {
}
addtask kernel_link_images after do_compile before do_strip
-do_strip() {
- if [ -n "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}" ]; then
- if ! (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux"); then
- bbwarn "image type(s) will not be stripped (not
supported): ${KERNEL_IMAGETYPES}"
- return
- fi
-
- cd ${B}
- headers=`"$CROSS_COMPILE"readelf -S
${KERNEL_OUTPUT_DIR}/vmlinux | \
- grep "^ \{1,\}\[[0-9 ]\{1,\}\] [^ ]" | \
- sed "s/^ \{1,\}\[[0-9 ]\{1,\}\] //" | \
- gawk '{print $1}'`
-
- for str in ${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}; do {
- if ! (echo "$headers" | grep -q "^$str$"); then
- bbwarn "Section not found: $str";
- fi
-
- "$CROSS_COMPILE"strip -s -R $str
${KERNEL_OUTPUT_DIR}/vmlinux
- }; done
-
- bbnote "KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping
sections:" \
- "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}"
- fi;
+python do_strip() {
+import shutil
+
+strip = d.getVar('STRIP')
+extra_sections = d.getVar('KERNEL_IMAGE_STRIP_EXTRA_SECTIONS')
+kernel_image = d.getVar('B') + "/" + d.getVar('KERNEL_OUTPUT_DIR') +
"/vmlinux"
+
+if (extra_sections and kernel_image.find('boot/vmlinux') != -1):
+kernel_image_stripped = kernel_image + ".stripped"
+shutil.copy2(kernel_image, kernel_image_stripped)
+oe.package.runstrip((kernel_image_stripped, 8, strip, extra_sections))
+bb.debug(1, "KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping
sections: " + \
+extra_sections)
}
do_strip[dirs] = "${B}"
@@ -768,7 +757,12 @@ kernel_do_deploy() {
for imageType in ${KERNEL_IMAGETYPES} ; do
baseName=$imageType-${KERNEL_IMAGE_NAME}
- install -m 0644 ${KERNEL_OUTPUT_DIR}/$imageType
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+
+ if [ -s ${KERNEL_OUTPUT_DIR}/$imageType.stripped ] ; then
+ install -m 0644
${KERNEL_OUTPUT_DIR}/$imageType.stripped
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+ else
+ install -m 0644 ${KERNEL_OUTPUT_DIR}/$imageType
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+ fi
if [ -n "${KERNEL_IMAGE_LINK_NAME}" ] ; then
ln -sf $baseName${KERNEL_IMAGE_BIN_EXT}
$deployDir/$imageType-${KERNEL_IMAGE_LINK_NAME}${KERNEL_IMAGE_BIN_EXT}
fi
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160493):
https://lists.openembedded.org/g/openembedded-core/message/160493
Mute This Topic: https://lists.openembedded.org/mt/88377978/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/3] create-spdx: Add kernel work-shared source
Since the kernel source is stored in work-shared, we need to add it
to the search path so the kernel and kernel-modules source code can
be found correctly.
Signed-off-by: Saul Wold
---
meta/classes/create-spdx.bbclass | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 0a4db80aba5..e1ff2bbb787 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -254,6 +254,7 @@ def add_package_sources_from_debug(d, package_doc,
spdx_package, package, packag
Path(d.getVar('PKGD')),
Path(d.getVar('STAGING_DIR_TARGET')),
Path(d.getVar('STAGING_DIR_NATIVE')),
+Path(d.getVar('STAGING_KERNEL_DIR')),
]
pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
@@ -275,7 +276,10 @@ def add_package_sources_from_debug(d, package_doc,
spdx_package, package, packag
for debugsrc in file_data["debugsrc"]:
ref_id = "NOASSERTION"
for search in debug_search_paths:
-debugsrc_path = search / debugsrc.lstrip("/")
+if debugsrc.startswith("/usr/src/kernel"):
+debugsrc_path = search /
debugsrc.replace('/usr/src/kernel/', '')
+else:
+debugsrc_path = search / debugsrc.lstrip("/")
if not debugsrc_path.exists():
continue
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160495):
https://lists.openembedded.org/g/openembedded-core/message/160495
Mute This Topic: https://lists.openembedded.org/mt/88377980/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/3] Add support for creating spdx data for kernel
This change set is ready for merging, I have tested with edgerouter, qemuppc and x86, in order to check different kernel types and strip options. This version creates a copy to be stripped and deployed in kernel.bbclass, if needed. The unstripped version is installed so the package.bbclass debuginfo / extended packagedata can be generated correctly. This is the data that's used by create-spdx. Sau! Saul Wold (3): create-spdx: Add kernel work-shared source package: Add support for kernel stripping kernel.bbclass: use common strip() meta/classes/create-spdx.bbclass | 6 - meta/classes/kernel.bbclass | 44 ++-- meta/classes/package.bbclass | 17 meta/lib/oe/package.py | 9 ++- 4 files changed, 37 insertions(+), 39 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#160492): https://lists.openembedded.org/g/openembedded-core/message/160492 Mute This Topic: https://lists.openembedded.org/mt/88377977/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [RFC PATCH v2 3/3] kernel.bbclass: use common strip()
On 1/12/22 06:30, Bruce Ashfield wrote:
On Tue, Jan 11, 2022 at 6:59 PM Saul Wold wrote:
Re-use the runstrip() code from oe.packaging, for the kernel
stripping process. Since runstrip() is python the kernel do_strip()
need to be converted to Python also. The stripped kernel image
will be used for deployment in do_deploy(). This will allow the
package.bbclass to split an unstripped kernel binary for the debug
information and extended packagedata. The extended package data is
used by create-spdx.
Signed-off-by: Saul Wold
---
meta/classes/kernel.bbclass | 44 -
1 file changed, 19 insertions(+), 25 deletions(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 7685c6921fa..b2f9e3c8071 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -700,30 +700,19 @@ do_kernel_link_images() {
}
addtask kernel_link_images after do_compile before do_strip
-do_strip() {
- if [ -n "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}" ]; then
- if ! (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux"); then
- bbwarn "image type(s) will not be stripped (not supported):
${KERNEL_IMAGETYPES}"
- return
- fi
-
- cd ${B}
- headers=`"$CROSS_COMPILE"readelf -S
${KERNEL_OUTPUT_DIR}/vmlinux | \
- grep "^ \{1,\}\[[0-9 ]\{1,\}\] [^ ]" | \
- sed "s/^ \{1,\}\[[0-9 ]\{1,\}\] //" | \
- gawk '{print $1}'`
-
- for str in ${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}; do {
- if ! (echo "$headers" | grep -q "^$str$"); then
- bbwarn "Section not found: $str";
- fi
We are losing this warning in the relocated / common routine. It looks
like something we could continue to detect and warn on, no ?
I guess do we care in this case? We still do the strip regardless, the
strip command does not complain about a missing section.
-
- "$CROSS_COMPILE"strip -s -R $str
${KERNEL_OUTPUT_DIR}/vmlinux
- }; done
-
- bbnote "KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping
sections:" \
- "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}"
- fi;
+python do_strip() {
+import shutil
+
+strip = d.getVar('STRIP')
+extra_sections = d.getVar('KERNEL_IMAGE_STRIP_EXTRA_SECTIONS')
+kernel_image = d.getVar('B') + "/" + d.getVar('KERNEL_OUTPUT_DIR') +
"/vmlinux"
+
+if (extra_sections is not None and kernel_image.find('boot/vmlinux') !=
-1):
Minor 'nit .. I'd probably just write that as: if extra_sections
(versus the is not None) .. but maybe you have it that way due to that
check not working.
+kernel_image_stripped = kernel_image + ".stripped"
+shutil.copy2(kernel_image, kernel_image_stripped)
+oe.package.runstrip((kernel_image_stripped, 8, strip, extra_sections))
+bb.note ("KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping sections:
" + \
+extra_sections)
In theory, we could move the debug log to the common routine when it
detects the extra section argument
Sure this can move to bb.debug(), will do that.
}
do_strip[dirs] = "${B}"
@@ -768,7 +757,12 @@ kernel_do_deploy() {
for imageType in ${KERNEL_IMAGETYPES} ; do
baseName=$imageType-${KERNEL_IMAGE_NAME}
- install -m 0644 ${KERNEL_OUTPUT_DIR}/$imageType
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+
+ if [ -s ${KERNEL_OUTPUT_DIR}/$imageType.stripped ] ; then
+ install -m 0644
${KERNEL_OUTPUT_DIR}/$imageType.stripped
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+ else
+ install -m 0644 ${KERNEL_OUTPUT_DIR}/$imageType
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+ fi
Is there a scenario when the ".stripped" image isn't available ?
Wouldn't that already be an error, and so the deploy should just be an
error, versus copying the non-stripped one ?
We only copy to .stripped when KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set,
so in the non-set case we will have the un-stripped image that needs to
get installed correctly.
Sau!
Bruce
if [ -n "${KERNEL_IMAGE_LINK_NAME}" ] ; then
ln -sf $baseName${KERNEL_IMAGE_BIN_EXT}
$deployDir/$imageType-${KERNEL_IMAGE_LINK_NAME}${KERNEL_IMAGE_BIN_EXT}
fi
--
2.31.1
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160490):
https://lists.openembedded.org/g/openembedded-core/message/160490
Mute This Topic: https://lists.openembedded.org/mt/88362607/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH v2 3/3] kernel.bbclass: use common strip()
Re-use the runstrip() code from oe.packaging, for the kernel
stripping process. Since runstrip() is python the kernel do_strip()
need to be converted to Python also. The stripped kernel image
will be used for deployment in do_deploy(). This will allow the
package.bbclass to split an unstripped kernel binary for the debug
information and extended packagedata. The extended package data is
used by create-spdx.
Signed-off-by: Saul Wold
---
meta/classes/kernel.bbclass | 44 -
1 file changed, 19 insertions(+), 25 deletions(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 7685c6921fa..b2f9e3c8071 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -700,30 +700,19 @@ do_kernel_link_images() {
}
addtask kernel_link_images after do_compile before do_strip
-do_strip() {
- if [ -n "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}" ]; then
- if ! (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux"); then
- bbwarn "image type(s) will not be stripped (not
supported): ${KERNEL_IMAGETYPES}"
- return
- fi
-
- cd ${B}
- headers=`"$CROSS_COMPILE"readelf -S
${KERNEL_OUTPUT_DIR}/vmlinux | \
- grep "^ \{1,\}\[[0-9 ]\{1,\}\] [^ ]" | \
- sed "s/^ \{1,\}\[[0-9 ]\{1,\}\] //" | \
- gawk '{print $1}'`
-
- for str in ${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}; do {
- if ! (echo "$headers" | grep -q "^$str$"); then
- bbwarn "Section not found: $str";
- fi
-
- "$CROSS_COMPILE"strip -s -R $str
${KERNEL_OUTPUT_DIR}/vmlinux
- }; done
-
- bbnote "KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping
sections:" \
- "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}"
- fi;
+python do_strip() {
+import shutil
+
+strip = d.getVar('STRIP')
+extra_sections = d.getVar('KERNEL_IMAGE_STRIP_EXTRA_SECTIONS')
+kernel_image = d.getVar('B') + "/" + d.getVar('KERNEL_OUTPUT_DIR') +
"/vmlinux"
+
+if (extra_sections is not None and kernel_image.find('boot/vmlinux') !=
-1):
+kernel_image_stripped = kernel_image + ".stripped"
+shutil.copy2(kernel_image, kernel_image_stripped)
+oe.package.runstrip((kernel_image_stripped, 8, strip, extra_sections))
+bb.note ("KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping
sections: " + \
+extra_sections)
}
do_strip[dirs] = "${B}"
@@ -768,7 +757,12 @@ kernel_do_deploy() {
for imageType in ${KERNEL_IMAGETYPES} ; do
baseName=$imageType-${KERNEL_IMAGE_NAME}
- install -m 0644 ${KERNEL_OUTPUT_DIR}/$imageType
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+
+ if [ -s ${KERNEL_OUTPUT_DIR}/$imageType.stripped ] ; then
+ install -m 0644
${KERNEL_OUTPUT_DIR}/$imageType.stripped
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+ else
+ install -m 0644 ${KERNEL_OUTPUT_DIR}/$imageType
$deployDir/$baseName${KERNEL_IMAGE_BIN_EXT}
+ fi
if [ -n "${KERNEL_IMAGE_LINK_NAME}" ] ; then
ln -sf $baseName${KERNEL_IMAGE_BIN_EXT}
$deployDir/$imageType-${KERNEL_IMAGE_LINK_NAME}${KERNEL_IMAGE_BIN_EXT}
fi
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160475):
https://lists.openembedded.org/g/openembedded-core/message/160475
Mute This Topic: https://lists.openembedded.org/mt/88362607/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH v2 2/3] package: Add support for kernel stripping
Extend runstrip() to accept additional argument to enable
sharing it with the kernel do_strip() to that
KERNEL_IMAGE_STRIP_EXTRA_SECTIONS can be passed.
Since is_elf() understands kernel modules there is no need to keep a
seperate list for kernmodules or hardcode the values to runstrip.
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 21 +++--
meta/lib/oe/package.py | 7 +--
2 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 09cd376f4af..794996e6d6d 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -390,10 +390,6 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir,
debugappend, debugsrcdir,
dvar = d.getVar('PKGD')
objcopy = d.getVar("OBJCOPY")
-# We ignore kernel modules, we don't generate debug info files.
-if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
-return (file, sources)
-
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
origmode = os.stat(file)[stat.ST_MODE]
@@ -1122,7 +1118,6 @@ python split_and_strip_files () {
#
elffiles = {}
symlinks = {}
-kernmods = []
staticlibs = []
inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir"))
@@ -1145,9 +1140,6 @@ python split_and_strip_files () {
if file in skipfiles:
continue
-if file.endswith(".ko") and file.find("/lib/modules/") != -1:
-kernmods.append(file)
-continue
if oe.package.is_static_lib(file):
staticlibs.append(file)
continue
@@ -1164,8 +1156,11 @@ python split_and_strip_files () {
if not s:
continue
# Check its an executable
-if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-or ((file.startswith(libdir) or
file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
+if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) \
+or (s[stat.ST_MODE] & stat.S_IXOTH) \
+or ((file.startswith(libdir) or
file.startswith(baselibdir)) \
+and (".so" in f or ".node" in f)) \
+or (f.startswith('vmlinux') or ".ko" in f):
if cpath.islink(file):
checkelflinks[file] = ltarget
@@ -1311,12 +1306,10 @@ python split_and_strip_files () {
for file in elffiles:
elf_file = int(elffiles[file])
#bb.note("Strip %s" % file)
-sfiles.append((file, elf_file, strip))
-for f in kernmods:
-sfiles.append((f, 16, strip))
+sfiles.append((file, elf_file, strip, ''))
if (d.getVar('PACKAGE_STRIP_STATIC') == '1' or
d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
for f in staticlibs:
-sfiles.append((f, 16, strip))
+sfiles.append((f, 16, strip, ''))
oe.utils.multiprocess_launch(oe.package.runstrip, sfiles, d)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index dd700cbb0c9..7842a614e9f 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -16,7 +16,7 @@ def runstrip(arg):
# 8 - shared library
# 16 - kernel module
-(file, elftype, strip) = arg
+(file, elftype, strip, extra_strip_sections) = arg
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
@@ -40,6 +40,9 @@ def runstrip(arg):
# shared or executable:
elif elftype & 8 or elftype & 4:
stripcmd.extend(["--remove-section=.comment",
"--remove-section=.note"])
+if file.find("boot/vmlinux") != -1 and extra_strip_sections != '':
+for section in extra_strip_sections.split():
+stripcmd.extend(["--remove-section=" + section])
stripcmd.append(file)
bb.debug(1, "runstrip: %s" % stripcmd)
@@ -172,7 +175,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir,
d, qa_already_stripp
sfiles = []
for file in elffiles:
elf_file = int(elffiles[file])
-sfiles.append((file, elf_file, strip_cmd))
+sfiles.append((file, elf_file, strip_cmd, ''))
oe.utils.multiprocess_launch(runstrip, sfiles, d)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160474):
https://lists.openembedded.org/g/openembedded-core/message/160474
Mute This Topic: https://lists.openembedded.org/mt/88362605/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH v2 0/3] Extend create-spdx to build kernel spdx info
This second change set, enables the kernel:do_strip() to share the oe.package:runstrip() method. A copy of the kernel is made for the kernel strip processing which is ultimately deployed, while the unstripped kernel image can be used by package.bbclass to generate the extended packagedata (debug info). The extended packagedata is also used to genrate the SPDX data. This change also needs to strip out the "/usr/src/kernel" path since the kernel uses work-shared, which is added to the search path. I have tested this with edgerouter which uses the KERNEL_IMAGE_STRIP_EXTRA_SECTIONS variable If this looks good, I will send a non-RFC patch set tomorrow. Sau! Saul Wold (3): create-spdx: Add kernel work-shared source package: Add support for kernel stripping kernel.bbclass: use common strip() meta/classes/create-spdx.bbclass | 6 - meta/classes/kernel.bbclass | 44 ++-- meta/classes/package.bbclass | 21 +-- meta/lib/oe/package.py | 7 +++-- 4 files changed, 36 insertions(+), 42 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#160472): https://lists.openembedded.org/g/openembedded-core/message/160472 Mute This Topic: https://lists.openembedded.org/mt/88362603/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH v2 1/3] create-spdx: Add kernel work-shared source
Since the kernel source is stored in work-shared, we need to add it
to the search path so the kernel and kernel-modules source code can
be found correctly.
Signed-off-by: Saul Wold
---
meta/classes/create-spdx.bbclass | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 0a4db80aba5..e1ff2bbb787 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -254,6 +254,7 @@ def add_package_sources_from_debug(d, package_doc,
spdx_package, package, packag
Path(d.getVar('PKGD')),
Path(d.getVar('STAGING_DIR_TARGET')),
Path(d.getVar('STAGING_DIR_NATIVE')),
+Path(d.getVar('STAGING_KERNEL_DIR')),
]
pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
@@ -275,7 +276,10 @@ def add_package_sources_from_debug(d, package_doc,
spdx_package, package, packag
for debugsrc in file_data["debugsrc"]:
ref_id = "NOASSERTION"
for search in debug_search_paths:
-debugsrc_path = search / debugsrc.lstrip("/")
+if debugsrc.startswith("/usr/src/kernel"):
+debugsrc_path = search /
debugsrc.replace('/usr/src/kernel/', '')
+else:
+debugsrc_path = search / debugsrc.lstrip("/")
if not debugsrc_path.exists():
continue
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160473):
https://lists.openembedded.org/g/openembedded-core/message/160473
Mute This Topic: https://lists.openembedded.org/mt/88362604/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [RFC PATCH 2/3] kernel.bbclass: remove do_strip() method
On 1/11/22 03:27, Richard Purdie wrote:
On Mon, 2022-01-10 at 16:42 -0800, Saul Wold wrote:
On 1/8/22 11:27, Bruce Ashfield wrote:
On Fri, Jan 7, 2022 at 6:19 PM Richard Purdie
mailto:richard.pur...@linuxfoundation.org>> wrote:
On Fri, 2022-01-07 at 13:24 -0800, Saul Wold wrote:
> Move the do_strip() functionality to a more common location in the
> package split_and_strip_files() flow. This makes it possible for the
> extended packaging data to be generated correctly for the kernel and
> kernel modules. The KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is reused in
> the runstrip() part of package stripping.
>
> Signed-off-by: Saul Wold mailto:saul.w...@windriver.com>>
> ---
> meta/classes/kernel.bbclass | 35 +++
> 1 file changed, 3 insertions(+), 32 deletions(-)
As I mentioned in previous mails, I think this will mean the
do_deploy output
will change and become unstripped which may cause a problem for some
platforms.
We likely need to keep a strip task but call the new strip function
from it?
I personally think this is a reasonable direction apart from that
though, at
least in principle.
Agreed. It looks ok to me as well, with us making sure that do_deploy
can remain as it was.
I have been looking into this and realized it's maybe a little more
complex since the do_sizecheck() needs to run also before do_deploy.
Currently, I believe the ordering is:
do_compile()
do_compile_kernelmodules()
do_kernel_linux_images()
do_strip()
do_sizecheck()
do_install
do_package()
do_packagedata()
do_deploy()
If I move the strip() process to package, I also have to move when
do_sizecheck() runs. As you mention we need to keep what gets deployed
to be the same with and without the package split_and_strip_files().
I understand what your talking about having the vmlinux image (which is
the one getting stripped) is the same before and after this changeset, I
don't think it's an issue of the deployed vmlinux being un-stripped but
the vmlinux being stripped in the deploy dir.
We can't really call the package.bbclass strip from the kernel bbclass
since it's part of the package process and called in a multiprocess() way.
This bit puzzles me. Why can't we manually call the function from kernel
bbclass? It might not be particularly nice given the multiprocess bits but I'm
sure it can be done and likely the lesser of several evils. If it works in a
multiprocess context, it should work without? (the other way around is much
harder).
I might not of explained myself well, since I did not mention the
gathering of debug info (extended packagedata) occurs during the
do_package() phase. Yes, I can use the runstrip() function from
oe.package in kernel.bbclass, but it still means the kernel image is
stripped by the time it gets to the do_package when we want to gather
the debuginfo, but can't from the already stripped kernel image.
It's a task ordering issue in combination with what happens in the
split_and_strip_files() to get the extended packagedata that the
create-spdx bbclass then uses.
The kernel do_strip() happens on the vmlinux image in B, while the
package stripping happens in the PKGD directory, while do_deploy goes
back to B for the kernel image (KERNEL_IMAGETTYPES) which might not be
the vmlinux. It's a twisty little process with back and forth. I think
this is where I got a little confused also, I thought do_deploy was from
either D (WORKDIR/image) until I re-read the run.do_deploy script
itself! (my bad).
I could add special handling in the kernel to do the strip/split
Hope that makes better sense.
Sau!
Cheers,
Richard
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160464):
https://lists.openembedded.org/g/openembedded-core/message/160464
Mute This Topic: https://lists.openembedded.org/mt/88271781/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [RFC PATCH 2/3] kernel.bbclass: remove do_strip() method
On 1/8/22 11:27, Bruce Ashfield wrote: On Fri, Jan 7, 2022 at 6:19 PM Richard Purdie <mailto:richard.pur...@linuxfoundation.org>> wrote: On Fri, 2022-01-07 at 13:24 -0800, Saul Wold wrote: > Move the do_strip() functionality to a more common location in the > package split_and_strip_files() flow. This makes it possible for the > extended packaging data to be generated correctly for the kernel and > kernel modules. The KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is reused in > the runstrip() part of package stripping. > > Signed-off-by: Saul Wold mailto:saul.w...@windriver.com>> > --- > meta/classes/kernel.bbclass | 35 +++ > 1 file changed, 3 insertions(+), 32 deletions(-) As I mentioned in previous mails, I think this will mean the do_deploy output will change and become unstripped which may cause a problem for some platforms. We likely need to keep a strip task but call the new strip function from it? I personally think this is a reasonable direction apart from that though, at least in principle. Agreed. It looks ok to me as well, with us making sure that do_deploy can remain as it was. I have been looking into this and realized it's maybe a little more complex since the do_sizecheck() needs to run also before do_deploy. Currently, I believe the ordering is: do_compile() do_compile_kernelmodules() do_kernel_linux_images() do_strip() do_sizecheck() do_install do_package() do_packagedata() do_deploy() If I move the strip() process to package, I also have to move when do_sizecheck() runs. As you mention we need to keep what gets deployed to be the same with and without the package split_and_strip_files(). I understand what your talking about having the vmlinux image (which is the one getting stripped) is the same before and after this changeset, I don't think it's an issue of the deployed vmlinux being un-stripped but the vmlinux being stripped in the deploy dir. We can't really call the package.bbclass strip from the kernel bbclass since it's part of the package process and called in a multiprocess() way. One thought is to have a copy of the vmlinux that gets packaged and stripped via the package process vs the vmlinux binary that gets deployed and optionally stripped when KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set. Maybe we can talk though this during the Tech call Tuesday. Sau! Bruce Cheers, Richard -- - Thou shalt not follow the NULL pointer, for chaos and madness await thee at its end - "Use the force Harry" - Gandalf, Star Trek II -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#160382): https://lists.openembedded.org/g/openembedded-core/message/160382 Mute This Topic: https://lists.openembedded.org/mt/88271781/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH 2/3] kernel.bbclass: remove do_strip() method
Move the do_strip() functionality to a more common location in the
package split_and_strip_files() flow. This makes it possible for the
extended packaging data to be generated correctly for the kernel and
kernel modules. The KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is reused in
the runstrip() part of package stripping.
Signed-off-by: Saul Wold
---
meta/classes/kernel.bbclass | 35 +++
1 file changed, 3 insertions(+), 32 deletions(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 7685c6921fa..30e67abb936 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -421,7 +421,7 @@ do_compile_kernelmodules() {
bbnote "no modules to compile"
fi
}
-addtask compile_kernelmodules after do_compile before do_strip
+addtask compile_kernelmodules after do_compile before do_sizecheck
kernel_do_install() {
#
@@ -698,36 +698,7 @@ do_kernel_link_images() {
ln -sf ../../../vmlinux.64
fi
}
-addtask kernel_link_images after do_compile before do_strip
-
-do_strip() {
- if [ -n "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}" ]; then
- if ! (echo "${KERNEL_IMAGETYPES}" | grep -wq "vmlinux"); then
- bbwarn "image type(s) will not be stripped (not
supported): ${KERNEL_IMAGETYPES}"
- return
- fi
-
- cd ${B}
- headers=`"$CROSS_COMPILE"readelf -S
${KERNEL_OUTPUT_DIR}/vmlinux | \
- grep "^ \{1,\}\[[0-9 ]\{1,\}\] [^ ]" | \
- sed "s/^ \{1,\}\[[0-9 ]\{1,\}\] //" | \
- gawk '{print $1}'`
-
- for str in ${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}; do {
- if ! (echo "$headers" | grep -q "^$str$"); then
- bbwarn "Section not found: $str";
- fi
-
- "$CROSS_COMPILE"strip -s -R $str
${KERNEL_OUTPUT_DIR}/vmlinux
- }; done
-
- bbnote "KERNEL_IMAGE_STRIP_EXTRA_SECTIONS is set, stripping
sections:" \
- "${KERNEL_IMAGE_STRIP_EXTRA_SECTIONS}"
- fi;
-}
-do_strip[dirs] = "${B}"
-
-addtask strip before do_sizecheck after do_kernel_link_images
+addtask kernel_link_images after do_compile before do_sizecheck
# Support checking the kernel size since some kernels need to reside in
partitions
# with a fixed length or there is a limit in transferring the kernel to memory.
@@ -755,7 +726,7 @@ do_sizecheck() {
}
do_sizecheck[dirs] = "${B}"
-addtask sizecheck before do_install after do_strip
+addtask sizecheck before do_install after do_kernel_link_images
inherit kernel-artifact-names
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160269):
https://lists.openembedded.org/g/openembedded-core/message/160269
Mute This Topic: https://lists.openembedded.org/mt/88271781/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH 3/3] package: Add support for kernel stripping
This moves the KERNEL_IMAGE_STRIP_EXTRA_SECTIONS from kernel.bbclass
to the split_and_strip_files() flow. Since the multiprocess_launch is
used here the variable needs to be passed as an argument.
Since is_elf() understands kernel modules there is no need to keep a
seperate list for kernmodules or hardcode the values to runstrip.
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 23 ---
meta/lib/oe/package.py | 7 +--
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 09cd376f4af..2f6863be7ac 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -1122,7 +1122,6 @@ python split_and_strip_files () {
#
elffiles = {}
symlinks = {}
-kernmods = []
staticlibs = []
inodes = {}
libdir = os.path.abspath(dvar + os.sep + d.getVar("libdir"))
@@ -1145,9 +1144,6 @@ python split_and_strip_files () {
if file in skipfiles:
continue
-if file.endswith(".ko") and file.find("/lib/modules/") != -1:
-kernmods.append(file)
-continue
if oe.package.is_static_lib(file):
staticlibs.append(file)
continue
@@ -1164,8 +1160,11 @@ python split_and_strip_files () {
if not s:
continue
# Check its an executable
-if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-or ((file.startswith(libdir) or
file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
+if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) \
+or (s[stat.ST_MODE] & stat.S_IXOTH) \
+or ((file.startswith(libdir) or
file.startswith(baselibdir)) \
+and (".so" in f or ".node" in f)) \
+or (f.startswith('vmlinux') or ".ko" in f):
if cpath.islink(file):
checkelflinks[file] = ltarget
@@ -1310,13 +1309,15 @@ python split_and_strip_files () {
sfiles = []
for file in elffiles:
elf_file = int(elffiles[file])
-#bb.note("Strip %s" % file)
-sfiles.append((file, elf_file, strip))
-for f in kernmods:
-sfiles.append((f, 16, strip))
+bb.note("Strip %s" % file)
+extra_sections = d.getVar('KERNEL_IMAGE_STRIP_EXTRA_SECTIONS')
+if (extra_sections is not None and file.find('boot/vmlinux')):
+sfiles.append((file, elf_file, strip, extra_sections))
+else:
+sfiles.append((file, elf_file, strip, ''))
if (d.getVar('PACKAGE_STRIP_STATIC') == '1' or
d.getVar('PACKAGE_DEBUG_STATIC_SPLIT') == '1'):
for f in staticlibs:
-sfiles.append((f, 16, strip))
+sfiles.append((f, 16, strip, ''))
oe.utils.multiprocess_launch(oe.package.runstrip, sfiles, d)
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index dd700cbb0c9..47bc85a4266 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -16,7 +16,7 @@ def runstrip(arg):
# 8 - shared library
# 16 - kernel module
-(file, elftype, strip) = arg
+(file, elftype, strip, extra_strip_sections) = arg
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
@@ -40,6 +40,9 @@ def runstrip(arg):
# shared or executable:
elif elftype & 8 or elftype & 4:
stripcmd.extend(["--remove-section=.comment",
"--remove-section=.note"])
+if "boot/vmlinux" in file and extra_strip_sections != '':
+for section in extra_strip_sections.split():
+stripcmd.extend(["--remove-section=" + section])
stripcmd.append(file)
bb.debug(1, "runstrip: %s" % stripcmd)
@@ -172,7 +175,7 @@ def strip_execs(pn, dstdir, strip_cmd, libdir, base_libdir,
d, qa_already_stripp
sfiles = []
for file in elffiles:
elf_file = int(elffiles[file])
-sfiles.append((file, elf_file, strip_cmd))
+sfiles.append((file, elf_file, strip_cmd, ''))
oe.utils.multiprocess_launch(runstrip, sfiles, d)
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160267):
https://lists.openembedded.org/g/openembedded-core/message/160267
Mute This Topic: https://lists.openembedded.org/mt/88271779/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH 1/3] create-spdx: Add kernel work-shared source
Since the kernel source is stored in work-shared, we need to add it
to the search path so the kernel and kernel-modules source code can
be found correctly.
Signed-off-by: Saul Wold
---
meta/classes/create-spdx.bbclass | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 0a4db80aba5..e1ff2bbb787 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -254,6 +254,7 @@ def add_package_sources_from_debug(d, package_doc,
spdx_package, package, packag
Path(d.getVar('PKGD')),
Path(d.getVar('STAGING_DIR_TARGET')),
Path(d.getVar('STAGING_DIR_NATIVE')),
+Path(d.getVar('STAGING_KERNEL_DIR')),
]
pkg_data = oe.packagedata.read_subpkgdata_extended(package, d)
@@ -275,7 +276,10 @@ def add_package_sources_from_debug(d, package_doc,
spdx_package, package, packag
for debugsrc in file_data["debugsrc"]:
ref_id = "NOASSERTION"
for search in debug_search_paths:
-debugsrc_path = search / debugsrc.lstrip("/")
+if debugsrc.startswith("/usr/src/kernel"):
+debugsrc_path = search /
debugsrc.replace('/usr/src/kernel/', '')
+else:
+debugsrc_path = search / debugsrc.lstrip("/")
if not debugsrc_path.exists():
continue
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160268):
https://lists.openembedded.org/g/openembedded-core/message/160268
Mute This Topic: https://lists.openembedded.org/mt/88271780/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [RFC PATCH 0/3] Extend create-spdx to build kernel spdx info
This set of changes moves the kernel:do_strip() functionality to the packaging code related to split_and_strip_files(). The code checks the KENREL_IMAGE_STRIP_EXTRA_SECTIONS for the kernel specific path. This also needs to strip out the "/usr/src/kernel" path since the kernel uses work-shared, which is added to the search path. I am still working on testing this, but want to get more eyes on it. I am most concerned with the change to package.bbclass to handle the kernel and modules. As mentioned in the other patch thread, there was some concern about the stripping being too agressive. I am guessing that part of the change might need some tuning. Thanks Sau! Saul Wold (3): create-spdx: Add kernel work-shared source kernel.bbclass: remove do_strip() method package: Add support for kernel stripping meta/classes/create-spdx.bbclass | 6 +- meta/classes/kernel.bbclass | 35 +++- meta/classes/package.bbclass | 23 +++-- meta/lib/oe/package.py | 7 +-- 4 files changed, 25 insertions(+), 46 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#160270): https://lists.openembedded.org/g/openembedded-core/message/160270 Mute This Topic: https://lists.openembedded.org/mt/88271783/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2] perl: Enable threading
From: Saul Wold
When the tranisiton to perl-cross occured, the threading define
seems to have been missed. The perl tests for threading where
simply skipped, so there was no direct failures. This was verified
by running perl ptest before and after the change to see PASS vs SKIP
results of threaded related tests.
NOTE: Perl officially discourges the use of threads, so this
functionality maybe depercated in the future [0][1]
v2: adds the usethreads to native and nativesdk. This was tested by
builing postresql and rrdtool which use perl and automake.
[0] https://perldoc.perl.org/5.34.0-RC2/threads#WARNING
[1] https://perldoc.perl.org/perlpolicy#discouraged
Signed-off-by: Saul Wold
---
meta/recipes-devtools/perl/perl_5.34.0.bb | 3 +++
1 file changed, 3 insertions(+)
diff --git a/meta/recipes-devtools/perl/perl_5.34.0.bb
b/meta/recipes-devtools/perl/perl_5.34.0.bb
index a6ae80f07e1..e4bcfe3ce69 100644
--- a/meta/recipes-devtools/perl/perl_5.34.0.bb
+++ b/meta/recipes-devtools/perl/perl_5.34.0.bb
@@ -53,6 +53,7 @@ do_configure:class-target() {
./configure --prefix=${prefix} --libdir=${libdir} \
--target=${TARGET_SYS} \
-Duseshrplib \
+-Dusethreads \
-Dsoname=libperl.so.5 \
-Dvendorprefix=${prefix} \
-Darchlibexp=${STAGING_LIBDIR}/perl5/${PV}/${TARGET_ARCH}-linux \
@@ -80,6 +81,7 @@ do_configure:class-nativesdk() {
./configure --prefix=${prefix} \
--target=${TARGET_SYS} \
-Duseshrplib \
+-Dusethreads \
-Dsoname=libperl.so.5 \
-Dvendorprefix=${prefix} \
-Darchlibexp=${STAGING_LIBDIR}/perl5/${PV}/${TARGET_ARCH}-linux \
@@ -94,6 +96,7 @@ do_configure:class-native() {
./configure --prefix=${prefix} \
-Dbin=${bindir}/perl-native \
-Duseshrplib \
+-Dusethreads \
-Dsoname=libperl.so.5 \
-Dvendorprefix=${prefix} \
-Ui_xlocale \
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160244):
https://lists.openembedded.org/g/openembedded-core/message/160244
Mute This Topic: https://lists.openembedded.org/mt/88246693/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH] perl: Enable threading
On 1/6/22 10:22, Khem Raj wrote:
On Thu, Dec 30, 2021 at 1:24 PM Saul Wold wrote:
When the tranisiton to perl-cross occured, the threading define
seems to have been missed. The perl tests for threading where
simply skipped, so there was no direct failures. This was verified
by running perl ptest before and after the change to see PASS vs SKIP
results of threaded related tests.
Do we need this for perl-native too ? I am seeing failures like below
https://errors.yoctoproject.org/Errors/Details/621556/
https://errors.yoctoproject.org/Errors/Details/621555/
can you take a look ?
Yes, I am looking into it. RP mentioned it earlier today.
Yes, I guess I just enabled for the target and not native or nativesdk,
building and testing now.
Sau!
Signed-off-by: Saul Wold
---
meta/recipes-devtools/perl/perl_5.34.0.bb | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/recipes-devtools/perl/perl_5.34.0.bb
b/meta/recipes-devtools/perl/perl_5.34.0.bb
index a6ae80f07e1..3306a093692 100644
--- a/meta/recipes-devtools/perl/perl_5.34.0.bb
+++ b/meta/recipes-devtools/perl/perl_5.34.0.bb
@@ -53,6 +53,7 @@ do_configure:class-target() {
./configure --prefix=${prefix} --libdir=${libdir} \
--target=${TARGET_SYS} \
-Duseshrplib \
+-Dusethreads \
-Dsoname=libperl.so.5 \
-Dvendorprefix=${prefix} \
-Darchlibexp=${STAGING_LIBDIR}/perl5/${PV}/${TARGET_ARCH}-linux \
--
2.31.1
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160239):
https://lists.openembedded.org/g/openembedded-core/message/160239
Mute This Topic: https://lists.openembedded.org/mt/88042607/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 2/2] package.bbclass: don't skip kernel and kernel modules
On 1/5/22 09:30, Bruce Ashfield wrote:
On Wed, Jan 5, 2022 at 12:07 PM Richard Purdie
wrote:
On Tue, 2022-01-04 at 14:07 -0800, Saul Wold wrote:
On 12/22/21 01:09, Richard Purdie wrote:
On Tue, 2021-12-21 at 11:08 -0800, Saul Wold wrote:
Stop ignoring or skipping the kernel and kernel modules code in the
split debug and striping functions, this will allow create_spdx to
process the kernel and modules.
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 84eafbd529..4b7fe4f1e1 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -390,10 +390,6 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir,
debugappend, debugsrcdir,
dvar = d.getVar('PKGD')
objcopy = d.getVar("OBJCOPY")
-# We ignore kernel modules, we don't generate debug info files.
-if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
-return (file, sources)
-
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
origmode = os.stat(file)[stat.ST_MODE]
@@ -1147,7 +1143,7 @@ python split_and_strip_files () {
if file.endswith(".ko") and file.find("/lib/modules/") != -1:
kernmods.append(file)
-continue
+
if oe.package.is_static_lib(file):
staticlibs.append(file)
continue
@@ -1165,7 +1161,7 @@ python split_and_strip_files () {
continue
# Check its an executable
if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-or ((file.startswith(libdir) or file.startswith(baselibdir)) and
(".so" in f or ".node" in f)):
+or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in
f or ".node" in f)) or (f.startswith('vmlinux') or ".ko" in f):
if cpath.islink(file):
checkelflinks[file] = ltarget
edgerouter:
https://autobuilder.yoctoproject.org/typhoon/#/builders/62/builds/4513
https://autobuilder.yoctoproject.org/typhoon/#/builders/111/builds/2507/steps/11/logs/stdio
So I have been digging into this and it seems that an option was added a
decade ago or so to strip the kernel/vmlinux when it's too big, this was
done for at least the routerstationpro according to bug #3515 [0], and
persists with the edgerouter, although I am not sure if it would still
actually be required as the edgerouter also uses the
KERNEL_ALT_IMAGETYPE to create a smaller binary kernel image.
The change I proposed causes the all kernels to be stripped all the time
as part of the split_and_strip_files(). As I see it there few different
options:
1) Set KERNEL_IMAGE_EXTRA_STRIP_SECTIONS = "" in create_spdx.bbclass
- This solves the problem with create_spdx.bbclass is in use, but not
the general case
I don't think I like this as it is a side effect that isn't obvious or expected.
2) Remove the KERNEL_IMAGE_EXTRA_STRIP_SECTIONS from edgerouter.conf
- Will solve the edgerouter case but may not solve other usages
unknown to me.
- Does anyone know of other machines/layers usage of this variable?
3) deprecate the kernel.bbclass:do_strip function in favor of using the
split_and_strip_files() of package.bbclass
I know Bruce has said he doesn't like this, however stepping back, these issues
were from a time our stripping code was young and evolving. If we can
standardise and have it all work together well in one set of functions, I think
that is worth looking at. I'd prefer the kernel wasn't a special case if it no
longer needs to be.
That said, I don't remember the details of why we did this.
There's a middle ground of debug being possible, and some sections
removed to keep the footprint a bit lower. There were also some
unwinders, etc, that didn't work when everything was stripped and
split into debug. The stripping was too aggressive, and removed some
sections that were required.
While I can't exactly point to the use cases for it now, with the 5K
options in the kernel, they haven't all been removed, and I'd be very
hesitant to remove the capability completely.
I think this makes the most sense after thinking about it also, having
one place where the striping occurs in runstrip() in lib/oe/package.py,
seems reasonable. The one neck to ring as it were.
We can extend the is_elf() types to add vmlinux and use the
KERNEL_IMAGE_EXTRA_STRIP_SECTIONS there. So this could deprecate the
do_strip() from the kernel.bbclass and keep the behavior.
Sau!
Bruce
4) Change error to warning in packaging.bbclass for
Re: [OE-core] [PATCH 2/2] package.bbclass: don't skip kernel and kernel modules
On 12/22/21 01:09, Richard Purdie wrote:
On Tue, 2021-12-21 at 11:08 -0800, Saul Wold wrote:
Stop ignoring or skipping the kernel and kernel modules code in the
split debug and striping functions, this will allow create_spdx to
process the kernel and modules.
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 84eafbd529..4b7fe4f1e1 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -390,10 +390,6 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir,
debugappend, debugsrcdir,
dvar = d.getVar('PKGD')
objcopy = d.getVar("OBJCOPY")
-# We ignore kernel modules, we don't generate debug info files.
-if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
-return (file, sources)
-
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
origmode = os.stat(file)[stat.ST_MODE]
@@ -1147,7 +1143,7 @@ python split_and_strip_files () {
if file.endswith(".ko") and file.find("/lib/modules/") != -1:
kernmods.append(file)
-continue
+
if oe.package.is_static_lib(file):
staticlibs.append(file)
continue
@@ -1165,7 +1161,7 @@ python split_and_strip_files () {
continue
# Check its an executable
if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-or ((file.startswith(libdir) or file.startswith(baselibdir)) and
(".so" in f or ".node" in f)):
+or ((file.startswith(libdir) or file.startswith(baselibdir)) and (".so" in
f or ".node" in f)) or (f.startswith('vmlinux') or ".ko" in f):
if cpath.islink(file):
checkelflinks[file] = ltarget
edgerouter:
https://autobuilder.yoctoproject.org/typhoon/#/builders/62/builds/4513
https://autobuilder.yoctoproject.org/typhoon/#/builders/111/builds/2507/steps/11/logs/stdio
So I have been digging into this and it seems that an option was added a
decade ago or so to strip the kernel/vmlinux when it's too big, this was
done for at least the routerstationpro according to bug #3515 [0], and
persists with the edgerouter, although I am not sure if it would still
actually be required as the edgerouter also uses the
KERNEL_ALT_IMAGETYPE to create a smaller binary kernel image.
The change I proposed causes the all kernels to be stripped all the time
as part of the split_and_strip_files(). As I see it there few different
options:
1) Set KERNEL_IMAGE_EXTRA_STRIP_SECTIONS = "" in create_spdx.bbclass
- This solves the problem with create_spdx.bbclass is in use, but not
the general case
2) Remove the KERNEL_IMAGE_EXTRA_STRIP_SECTIONS from edgerouter.conf
- Will solve the edgerouter case but may not solve other usages
unknown to me.
- Does anyone know of other machines/layers usage of this variable?
3) deprecate the kernel.bbclass:do_strip function in favor of using the
split_and_strip_files() of package.bbclass
4) Change error to warning in packaging.bbclass for the kernel only
- This would explain that a kernel image (vmlinux) is already
stripped and extended package data would not be available for for SPDX
creation.
RP, Bruce, Joshua: Thoughts?
Sau!
[0] https://bugzilla.yoctoproject.org/show_bug.cgi?id=3515
qemux86 musl cryptodev:
https://autobuilder.yoctoproject.org/typhoon/#/builders/64/builds/4512/steps/11/logs/stdio
qemux86-64 musl cryptodev:
https://autobuilder.yoctoproject.org/typhoon/#/builders/45/builds/4526
qemux86 cryptodev:
https://autobuilder.yoctoproject.org/typhoon/#/builders/52/builds/4476/steps/11/logs/stdio
I tried these and have not been able to reproduce the failure.
selftest failure in linux-yocto:
https://autobuilder.yoctoproject.org/typhoon/#/builders/87/builds/2981/steps/14/logs/stdio
(file truncated makes it sound like a race?)
stap kernel module failure:
https://autobuilder.yoctoproject.org/typhoon/#/builders/110/builds/3395/steps/13/logs/stdio
(race/intermittent?)
another kernel module race:
https://autobuilder.yoctoproject.org/typhoon/#/builders/63/builds/4480/steps/16/logs/stdio
Might need to try this on the AB once I resolve the kernel stripping
issue above.
Sau!
Cheers,
Richard
--
Sau!
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160172):
https://lists.openembedded.org/g/openembedded-core/message/160172
Mute This Topic: https://lists.openembedded.org/mt/87884056/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH] perl: Enable threading
When the tranisiton to perl-cross occured, the threading define
seems to have been missed. The perl tests for threading where
simply skipped, so there was no direct failures. This was verified
by running perl ptest before and after the change to see PASS vs SKIP
results of threaded related tests.
Signed-off-by: Saul Wold
---
meta/recipes-devtools/perl/perl_5.34.0.bb | 1 +
1 file changed, 1 insertion(+)
diff --git a/meta/recipes-devtools/perl/perl_5.34.0.bb
b/meta/recipes-devtools/perl/perl_5.34.0.bb
index a6ae80f07e1..3306a093692 100644
--- a/meta/recipes-devtools/perl/perl_5.34.0.bb
+++ b/meta/recipes-devtools/perl/perl_5.34.0.bb
@@ -53,6 +53,7 @@ do_configure:class-target() {
./configure --prefix=${prefix} --libdir=${libdir} \
--target=${TARGET_SYS} \
-Duseshrplib \
+-Dusethreads \
-Dsoname=libperl.so.5 \
-Dvendorprefix=${prefix} \
-Darchlibexp=${STAGING_LIBDIR}/perl5/${PV}/${TARGET_ARCH}-linux \
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#160066):
https://lists.openembedded.org/g/openembedded-core/message/160066
Mute This Topic: https://lists.openembedded.org/mt/88042607/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/2] package.bbclass: don't skip kernel and kernel modules
Stop ignoring or skipping the kernel and kernel modules code in the
split debug and striping functions, this will allow create_spdx to
process the kernel and modules.
Signed-off-by: Saul Wold
---
meta/classes/package.bbclass | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
diff --git a/meta/classes/package.bbclass b/meta/classes/package.bbclass
index 84eafbd529..4b7fe4f1e1 100644
--- a/meta/classes/package.bbclass
+++ b/meta/classes/package.bbclass
@@ -390,10 +390,6 @@ def splitdebuginfo(file, dvar, debugdir, debuglibdir,
debugappend, debugsrcdir,
dvar = d.getVar('PKGD')
objcopy = d.getVar("OBJCOPY")
-# We ignore kernel modules, we don't generate debug info files.
-if file.find("/lib/modules/") != -1 and file.endswith(".ko"):
-return (file, sources)
-
newmode = None
if not os.access(file, os.W_OK) or os.access(file, os.R_OK):
origmode = os.stat(file)[stat.ST_MODE]
@@ -1147,7 +1143,7 @@ python split_and_strip_files () {
if file.endswith(".ko") and file.find("/lib/modules/") != -1:
kernmods.append(file)
-continue
+
if oe.package.is_static_lib(file):
staticlibs.append(file)
continue
@@ -1165,7 +1161,7 @@ python split_and_strip_files () {
continue
# Check its an executable
if (s[stat.ST_MODE] & stat.S_IXUSR) or (s[stat.ST_MODE] &
stat.S_IXGRP) or (s[stat.ST_MODE] & stat.S_IXOTH) \
-or ((file.startswith(libdir) or
file.startswith(baselibdir)) and (".so" in f or ".node" in f)):
+or ((file.startswith(libdir) or
file.startswith(baselibdir)) and (".so" in f or ".node" in f)) or
(f.startswith('vmlinux') or ".ko" in f):
if cpath.islink(file):
checkelflinks[file] = ltarget
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159931):
https://lists.openembedded.org/g/openembedded-core/message/159931
Mute This Topic: https://lists.openembedded.org/mt/87884056/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/2] kernel: add -dbg package
Adding the dbg package allows the package bbclass to parse the
debug information which can then be used by the create_spdx bbclass
Signed-off-by: Saul Wold
---
meta/classes/kernel.bbclass | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meta/classes/kernel.bbclass b/meta/classes/kernel.bbclass
index 2d219cb5e5..bffe264380 100644
--- a/meta/classes/kernel.bbclass
+++ b/meta/classes/kernel.bbclass
@@ -625,7 +625,7 @@ EXPORT_FUNCTIONS do_compile do_install do_configure
# kernel-base becomes kernel-${KERNEL_VERSION}
# kernel-image becomes kernel-image-${KERNEL_VERSION}
-PACKAGES = "${KERNEL_PACKAGE_NAME} ${KERNEL_PACKAGE_NAME}-base
${KERNEL_PACKAGE_NAME}-vmlinux ${KERNEL_PACKAGE_NAME}-image
${KERNEL_PACKAGE_NAME}-dev ${KERNEL_PACKAGE_NAME}-modules"
+PACKAGES = "${KERNEL_PACKAGE_NAME} ${KERNEL_PACKAGE_NAME}-base
${KERNEL_PACKAGE_NAME}-vmlinux ${KERNEL_PACKAGE_NAME}-image
${KERNEL_PACKAGE_NAME}-dev ${KERNEL_PACKAGE_NAME}-modules
${KERNEL_PACKAGE_NAME}-dbg"
FILES:${PN} = ""
FILES:${KERNEL_PACKAGE_NAME}-base =
"${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.order
${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.builtin
${nonarch_base_libdir}/modules/${KERNEL_VERSION}/modules.builtin.modinfo"
FILES:${KERNEL_PACKAGE_NAME}-image = ""
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#159930):
https://lists.openembedded.org/g/openembedded-core/message/159930
Mute This Topic: https://lists.openembedded.org/mt/87884054/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/2] Enable package.bbclass to process kernel packages
These changes allow the package bbclass to process the kernel and kernel modules so that they that extended packaging info can be used by the create-spdx bbclass. Tested with qemux86-64 Saul Wold (2): kernel: add -dbg package package.bbclass: don't skip kernel and kernel modules meta/classes/kernel.bbclass | 2 +- meta/classes/package.bbclass | 8 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159929): https://lists.openembedded.org/g/openembedded-core/message/159929 Mute This Topic: https://lists.openembedded.org/mt/87884053/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 4/4] core: udev: udev-extraconf: rename mount.blacklist* to mount.blocklist.*
On 12/8/21 01:57, Eero Aaltonen wrote: On Mon, 2021-12-06 at 23:31 +, Richard Purdie via lists.openembedded.org wrote: On Mon, 2021-12-06 at 16:35 +0100, Quentin Schulz wrote: blocklist has a more obvious meaning than blacklist and is also not an issue wrt inclusivity, so let's use that naming instead. A "blocklist" with a filesystem is unfortunately confusing (a list of block numbers on the filesystem?). "ignorelist" or even "ignore-devices" may be better? (or skip) I offer "denylist". I have recently added a list of patches and files that have problematic language to the Inclusive Language Wiki [0], this is one of them, I had proposed mount.disallow So we can see that everyone has a different idea. Once we have an approved rename, we can revisit this patch. Thanks Sau! [0] https://wiki.yoctoproject.org/wiki/Inclusive_language#Patch_Files Cheers, Eero -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159793): https://lists.openembedded.org/g/openembedded-core/message/159793 Mute This Topic: https://lists.openembedded.org/mt/87542276/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [oe] Help with Inclusive Language in OpenEmbedded/Yocto Project
On 12/6/21 17:01, Jon Mason wrote: This email is a follow-up from the session held on Friday at the OpenEmbedded Developer's Virtual Meeting (see https://www.openembedded.org/wiki/OEDVM_Nov_2021) The session was not recorded, but the slides can be found at https://docs.google.com/presentation/d/146ueVVTMeA8JI43wqv5kFmdYEygqqmfGH0z1VRL2bDA/edit?usp=sharing The outcome from the discussion was that inclusive language changes are something that we want to accomplish in the kirkstone release timeframe (with an exception for the "master" branch name, which will be handled at a future date). There has already been a pass at collecting the needed changes at https://wiki.yoctoproject.org/wiki/Inclusive_language This is not as simple as a find/replace of offending words. There is a desire for backward compatibility or to provide some kind of "you want X, which is now Y" (which complicates things). The intention of this email is to see who is interested in helping out. Once we know how many people are available and what time frames, we can plan out a roadmap. So, please email me (or respond to this thread publicly) and I'll add you to the list. There will then be a follow-up zoom call in the next week or so to plan out the roadmap. I am interested in helping out also. Another low hanging item might be changing the names of patches that include the offensive terms like the following (which I will add to the wiki: meta-openembedded/meta-oe/recipes-graphics/lxdm/lxdm/0001-lxdm.conf.in-blacklist-root-for-release-images.patch meta-openembedded/meta-oe/recipes-support/multipath-tools/files/0022-RH-Remove-the-property-blacklist-exception-builtin.patch oe-core/meta/recipes-extended/tcp-wrappers/tcp-wrappers-7.6/11_tcpd_blacklist.patch oe-core/meta/recipes-core/udev/udev-extraconf/mount.blacklist Can't really rename this one or we rename it in oe-core but it gets named back on the installed system. meta-secure-core/meta-integrity/files/ima_signing_blacklist Same as above meta-secure-core/meta-efi-secure-boot/recipes-bsp/efitools/efitools/Fix-the-wrong-dependency-for-blacklist.esl.patch We would have to re-generate the patches to have the subject match the fixed language. Sau! We will document the roadmap and everything else on the YP wiki page above. Questions and comments are welcome, but not interested in debating the necessity or timeframe of this task. It has already been decided. Thanks, Jon -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#159739): https://lists.openembedded.org/g/openembedded-core/message/159739 Mute This Topic: https://lists.openembedded.org/mt/87751253/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/2] create-spdx: Protect against None from LICENSE_PATH
If LICENSE_PATH is not set, then the split() will fail on a NoneType.
Signed-off-by: Saul Wold
---
meta/classes/create-spdx.bbclass | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 1d5c8b3bc1..d0cc5b1ca2 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -92,7 +92,7 @@ def convert_license_to_spdx(lic, document, d, existing={}):
extracted_info.extractedText = "Software released to the public
domain"
elif name in available_licenses:
# This license can be found in COMMON_LICENSE_DIR or LICENSE_PATH
-for directory in [d.getVar('COMMON_LICENSE_DIR')] +
d.getVar('LICENSE_PATH').split():
+for directory in [d.getVar('COMMON_LICENSE_DIR')] +
(d.getVar('LICENSE_PATH') or '').split():
try:
with (Path(directory) / name).open(errors="replace") as f:
extracted_info.extractedText = f.read()
@@ -145,7 +145,6 @@ def convert_license_to_spdx(lic, document, d, existing={}):
return ' '.join(convert(l) for l in lic_split)
-
def process_sources(d):
pn = d.getVar('PN')
assume_provided = (d.getVar("ASSUME_PROVIDED") or "").split()
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158451):
https://lists.openembedded.org/g/openembedded-core/message/158451
Mute This Topic: https://lists.openembedded.org/mt/87125933/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/2] Couple of SPDX fixes
I was trying to build more layers such as meta-clang and found additional changes are needed to support other packages that are installed into work-shared. We might also need a similar patch for the archiver is this change is acceptable. Sau! Saul Wold (2): create_spdx: ensure is_work_shared() is uniqe create-spdx: Protect against None from LICENSE_PATH meta/classes/create-spdx.bbclass | 15 ++- 1 file changed, 6 insertions(+), 9 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158449): https://lists.openembedded.org/g/openembedded-core/message/158449 Mute This Topic: https://lists.openembedded.org/mt/87125931/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/2] create_spdx: ensure is_work_shared() is uniqe
There is a function with the same name is_work_shared() in the archiver class
this causes a conflict when both classes are included. Use work-shared as the
check in WORKDIR to allow for other packages beyond the kernel and gcc that
use a common shared-work source directory.
Signed-off-by: Saul Wold
---
meta/classes/create-spdx.bbclass | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index eb1d446f3f..1d5c8b3bc1 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -53,10 +53,8 @@ def recipe_spdx_is_native(d, recipe):
a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION")) and
a.comment == "isNative" for a in recipe.annotations)
-def is_work_shared(d):
-pn = d.getVar('PN')
-return bb.data.inherits_class('kernel', d) or pn.startswith('gcc-source')
-
+def is_work_shared_spdx(d):
+return bb.data.inherits_class('kernel', d) or ('work-shared' in
d.getVar('WORKDIR'))
python() {
import json
@@ -747,7 +745,7 @@ def spdx_get_src(d):
try:
# The kernel class functions require it to be on work-shared, so we
dont change WORKDIR
-if not is_work_shared(d):
+if not is_work_shared_spdx(d):
# Change the WORKDIR to make do_unpack do_patch run in another dir.
d.setVar('WORKDIR', spdx_workdir)
# Restore the original path to recipe's native sysroot (it's
relative to WORKDIR).
@@ -760,7 +758,7 @@ def spdx_get_src(d):
bb.build.exec_func('do_unpack', d)
# Copy source of kernel to spdx_workdir
-if is_work_shared(d):
+if is_work_shared_spdx(d):
d.setVar('WORKDIR', spdx_workdir)
d.setVar('STAGING_DIR_NATIVE', spdx_sysroot_native)
src_dir = spdx_workdir + "/" + d.getVar('PN')+ "-" +
d.getVar('PV') + "-" + d.getVar('PR')
@@ -776,7 +774,7 @@ def spdx_get_src(d):
shutils.rmtree(git_path)
# Make sure gcc and kernel sources are patched only once
-if not (d.getVar('SRC_URI') == "" or is_work_shared(d)):
+if not (d.getVar('SRC_URI') == "" or is_work_shared_spdx(d)):
bb.build.exec_func('do_patch', d)
# Some userland has no source.
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#158450):
https://lists.openembedded.org/g/openembedded-core/message/158450
Mute This Topic: https://lists.openembedded.org/mt/87125932/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 0/3] SPDX: Add annotations to relationship
On 11/16/21 8:39 AM, Saul Wold wrote: On 11/15/21 2:44 PM, Paul Eggleton wrote: On Tuesday, 9 November 2021 08:01:38 NZDT Saul Wold wrote: On 11/4/21 2:20 PM, Joshua Watt wrote: On 11/4/21 3:50 PM, Richard Purdie wrote: On Thu, 2021-11-04 at 15:45 -0500, Joshua Watt wrote: On 11/4/21 3:43 PM, Richard Purdie wrote: On Thu, 2021-11-04 at 20:00 +, Jose Quaresma wrote: Richard Purdie escreveu no dia quinta, 28/10/2021 à(s) 21:58: On Thu, 2021-10-28 at 08:47 -1000, Steve Sakoman wrote: On Tue, Oct 26, 2021 at 10:41 PM Jose Quaresma wrote: Hi all, There are any plans or is it possible to backport the SBOM/SPDX to the dunfell branch? I'm going to yield to Saul as to whether he thinks this is desirable/possible or not. The packagedata changes are pretty invasive unfortunately and likely not something you're going to want in dunfell sadly. Thanks for the clarification. I have been thinking a bit more about this. I did wonder if we should consider a mixin layer of some kind for it that could work with dunfell? We could host it, it is just a question of writing the mixin layer and maintaining it. I don't think it's going to be possible with a pure mixin layer, since it relies on the extended package data? I suspect that could perhaps be patched in through a layer though? You might choose to drop the compression piece or do it differently for the backport? I'm not sure if a layer could hook in well enough to get the data needed... maybe worth an experiment though Yeah, I am not sure an mixin could track the changes for package.bbclass With a backport, I would probably either use GZip compression or no compression. The zstd compression was designed as a drop in replacement for Gzip if we wanted to go that route. I will say that we did something similar with Hardknott for WRLinux, but did not propose it upstream as Hardknott was knot going to be supported longer term. Having the spdx class standalone with the correctly backported changes seems to be working FYI Andres and I have done this backport to dunfell - should I post it? That said, I did just take the hit on some of the invasive parts (e.g. LICENSE value changes). I think given regulatory requirements this is important for lots of folks, so we probably need to do something here. Happy to be part of it. Hi Paul, Andres: We talked about this during the Tech Call this morning and the consensus was that this work should be done in a mix-in style layer so that it could be used by multiple releases. The LICENSE value changes could be handled by a single file with LICENSE_ style overrides in the mix-in layer, or by a set of bbappends in the mix-in layer. minor correct: LIENCE_pn- Did you include the compression changes or convert that back to basic XZ compression? We realize that this make for more work, but it's the problem of backporting a feature to the release vs having the feature in a separate mix-in. Hope this is clear. Sau! Cheers Paul -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158367): https://lists.openembedded.org/g/openembedded-core/message/158367 Mute This Topic: https://lists.openembedded.org/mt/86616599/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 0/3] SPDX: Add annotations to relationship
On 11/15/21 2:44 PM, Paul Eggleton wrote: On Tuesday, 9 November 2021 08:01:38 NZDT Saul Wold wrote: On 11/4/21 2:20 PM, Joshua Watt wrote: On 11/4/21 3:50 PM, Richard Purdie wrote: On Thu, 2021-11-04 at 15:45 -0500, Joshua Watt wrote: On 11/4/21 3:43 PM, Richard Purdie wrote: On Thu, 2021-11-04 at 20:00 +, Jose Quaresma wrote: Richard Purdie escreveu no dia quinta, 28/10/2021 à(s) 21:58: On Thu, 2021-10-28 at 08:47 -1000, Steve Sakoman wrote: On Tue, Oct 26, 2021 at 10:41 PM Jose Quaresma wrote: Hi all, There are any plans or is it possible to backport the SBOM/SPDX to the dunfell branch? I'm going to yield to Saul as to whether he thinks this is desirable/possible or not. The packagedata changes are pretty invasive unfortunately and likely not something you're going to want in dunfell sadly. Thanks for the clarification. I have been thinking a bit more about this. I did wonder if we should consider a mixin layer of some kind for it that could work with dunfell? We could host it, it is just a question of writing the mixin layer and maintaining it. I don't think it's going to be possible with a pure mixin layer, since it relies on the extended package data? I suspect that could perhaps be patched in through a layer though? You might choose to drop the compression piece or do it differently for the backport? I'm not sure if a layer could hook in well enough to get the data needed... maybe worth an experiment though Yeah, I am not sure an mixin could track the changes for package.bbclass With a backport, I would probably either use GZip compression or no compression. The zstd compression was designed as a drop in replacement for Gzip if we wanted to go that route. I will say that we did something similar with Hardknott for WRLinux, but did not propose it upstream as Hardknott was knot going to be supported longer term. Having the spdx class standalone with the correctly backported changes seems to be working FYI Andres and I have done this backport to dunfell - should I post it? That said, I did just take the hit on some of the invasive parts (e.g. LICENSE value changes). I think given regulatory requirements this is important for lots of folks, so we probably need to do something here. Happy to be part of it. Hi Paul, Andres: We talked about this during the Tech Call this morning and the consensus was that this work should be done in a mix-in style layer so that it could be used by multiple releases. The LICENSE value changes could be handled by a single file with LICENSE_ style overrides in the mix-in layer, or by a set of bbappends in the mix-in layer. Did you include the compression changes or convert that back to basic XZ compression? We realize that this make for more work, but it's the problem of backporting a feature to the release vs having the feature in a separate mix-in. Hope this is clear. Sau! Cheers Paul -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#158366): https://lists.openembedded.org/g/openembedded-core/message/158366 Mute This Topic: https://lists.openembedded.org/mt/86616599/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
Re: [OE-core] [PATCH 0/3] SPDX: Add annotations to relationship
On 11/4/21 2:20 PM, Joshua Watt wrote: On 11/4/21 3:50 PM, Richard Purdie wrote: On Thu, 2021-11-04 at 15:45 -0500, Joshua Watt wrote: On 11/4/21 3:43 PM, Richard Purdie wrote: On Thu, 2021-11-04 at 20:00 +, Jose Quaresma wrote: Richard Purdie escreveu no dia quinta, 28/10/2021 à(s) 21:58: On Thu, 2021-10-28 at 08:47 -1000, Steve Sakoman wrote: On Tue, Oct 26, 2021 at 10:41 PM Jose Quaresma wrote: Hi all, There are any plans or is it possible to backport the SBOM/SPDX to the dunfell branch? I'm going to yield to Saul as to whether he thinks this is desirable/possible or not. The packagedata changes are pretty invasive unfortunately and likely not something you're going to want in dunfell sadly. Thanks for the clarification. I have been thinking a bit more about this. I did wonder if we should consider a mixin layer of some kind for it that could work with dunfell? We could host it, it is just a question of writing the mixin layer and maintaining it. I don't think it's going to be possible with a pure mixin layer, since it relies on the extended package data? I suspect that could perhaps be patched in through a layer though? You might choose to drop the compression piece or do it differently for the backport? I'm not sure if a layer could hook in well enough to get the data needed... maybe worth an experiment though Yeah, I am not sure an mixin could track the changes for package.bbclass With a backport, I would probably either use GZip compression or no compression. The zstd compression was designed as a drop in replacement for Gzip if we wanted to go that route. I will say that we did something similar with Hardknott for WRLinux, but did not propose it upstream as Hardknott was knot going to be supported longer term. Having the spdx class standalone with the correctly backported changes seems to be working Sau! Cheers, Richard -- Sau! -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#157987): https://lists.openembedded.org/g/openembedded-core/message/157987 Mute This Topic: https://lists.openembedded.org/mt/86616599/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/3] SPDX: Add annotations to relationship
Add annotations to relationships and refactor code to add create_annotation() function for code reuse. Ensure that "cross" recipes are factored into isNative also. v2: removed leftover and unused annotation per Joshua Sau! Saul Wold (3): spdx.py: Add annotation to relationship create-spdx: add create_annotation function create-spdx: cross recipes are native also classes/create-spdx.bbclass | 22 ++ lib/oe/spdx.py | 6 +- 2 files changed, 19 insertions(+), 9 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#157470): https://lists.openembedded.org/g/openembedded-core/message/157470 Mute This Topic: https://lists.openembedded.org/mt/86616599/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 1/3] spdx.py: Add annotation to relationship
Having annotations on relationship can provide additional information about the relationship such as how it was derived. Signed-off-by: Saul Wold --- lib/oe/spdx.py | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/oe/spdx.py b/lib/oe/spdx.py index 4416194..9e7ced5 100644 --- a/lib/oe/spdx.py +++ b/lib/oe/spdx.py @@ -196,6 +196,7 @@ class SPDXRelationship(SPDXObject): relatedSpdxElement = _String() relationshipType = _String() comment = _String() +annotations = _ObjectList(SPDXAnnotation) class SPDXExternalReference(SPDXObject): @@ -300,7 +301,7 @@ class SPDXDocument(SPDXObject): def from_json(cls, f): return cls(**json.load(f)) -def add_relationship(self, _from, relationship, _to, *, comment=None): +def add_relationship(self, _from, relationship, _to, *, comment=None, annotation=None): if isinstance(_from, SPDXObject): from_spdxid = _from.SPDXID else: @@ -320,6 +321,9 @@ class SPDXDocument(SPDXObject): if comment is not None: r.comment = comment +if annotation is not None: +r.annotations.append(annotation) + self.relationships.append(r) def find_by_spdxid(self, spdxid): -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#157469): https://lists.openembedded.org/g/openembedded-core/message/157469 Mute This Topic: https://lists.openembedded.org/mt/86618682/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 2/3] create-spdx: add create_annotation function
This allows code reuse and future usage with relationship annotations
Signed-off-by: Saul Wold
---
classes/create-spdx.bbclass | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/classes/create-spdx.bbclass b/classes/create-spdx.bbclass
index dd341db..72a9b55 100644
--- a/classes/create-spdx.bbclass
+++ b/classes/create-spdx.bbclass
@@ -35,6 +35,17 @@ def get_doc_namespace(d, doc):
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name,
str(uuid.uuid5(namespace_uuid, doc.name)))
+def create_annotation(d, comment):
+from datetime import datetime, timezone
+
+creation_time =
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+annotation = oe.spdx.SPDXAnnotation()
+annotation.annotationDate = creation_time
+annotation.annotationType = "OTHER"
+annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION"))
+annotation.comment = comment
+return annotation
+
def recipe_spdx_is_native(d, recipe):
return any(a.annotationType == "OTHER" and
a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION")) and
@@ -408,12 +419,7 @@ python do_create_spdx() {
recipe.versionInfo = d.getVar("PV")
recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
if bb.data.inherits_class("native", d):
-annotation = oe.spdx.SPDXAnnotation()
-annotation.annotationDate = creation_time
-annotation.annotationType = "OTHER"
-annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION"))
-annotation.comment = "isNative"
-recipe.annotations.append(annotation)
+recipe.annotations.append(create_annotation(d, "isNative"))
for s in d.getVar('SRC_URI').split():
if not s.startswith("file://"):
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157468):
https://lists.openembedded.org/g/openembedded-core/message/157468
Mute This Topic: https://lists.openembedded.org/mt/86618681/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH v2 3/3] create-spdx: cross recipes are native also
Recipes that inherit cross should also be categorized as isNative
Signed-off-by: Saul Wold
---
classes/create-spdx.bbclass | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/classes/create-spdx.bbclass b/classes/create-spdx.bbclass
index 72a9b55..e8f476f 100644
--- a/classes/create-spdx.bbclass
+++ b/classes/create-spdx.bbclass
@@ -418,7 +418,7 @@ python do_create_spdx() {
recipe.name = d.getVar("PN")
recipe.versionInfo = d.getVar("PV")
recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
-if bb.data.inherits_class("native", d):
+if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross",
d):
recipe.annotations.append(create_annotation(d, "isNative"))
for s in d.getVar('SRC_URI').split():
@@ -610,7 +610,7 @@ python do_create_runtime_spdx() {
deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
-is_native = bb.data.inherits_class("native", d)
+is_native = bb.data.inherits_class("native", d) or
bb.data.inherits_class("cross", d)
creation_time =
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157467):
https://lists.openembedded.org/g/openembedded-core/message/157467
Mute This Topic: https://lists.openembedded.org/mt/86618680/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 0/3] SPDX: Add annotations to relationship
Add annotations to relationships and refactor code to add create_annotation() function for code reuse. Ensure that "cross" recipes are factored into isNative also. Sau! Saul Wold (3): spdx.py: Add annotation to relationship create-spdx: add create_annotation function create-spdx: cross recipes are native also classes/create-spdx.bbclass | 23 +++ lib/oe/spdx.py | 6 +- 2 files changed, 20 insertions(+), 9 deletions(-) -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#157459): https://lists.openembedded.org/g/openembedded-core/message/157459 Mute This Topic: https://lists.openembedded.org/mt/86616599/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/3] create-spdx: add create_annotation function
This allows code reuse and future usage with relationship annotations
Signed-off-by: Saul Wold
---
classes/create-spdx.bbclass | 18 --
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/classes/create-spdx.bbclass b/classes/create-spdx.bbclass
index dd341db..72a9b55 100644
--- a/classes/create-spdx.bbclass
+++ b/classes/create-spdx.bbclass
@@ -35,6 +35,17 @@ def get_doc_namespace(d, doc):
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name,
str(uuid.uuid5(namespace_uuid, doc.name)))
+def create_annotation(d, comment):
+from datetime import datetime, timezone
+
+creation_time =
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+annotation = oe.spdx.SPDXAnnotation()
+annotation.annotationDate = creation_time
+annotation.annotationType = "OTHER"
+annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION"))
+annotation.comment = comment
+return annotation
+
def recipe_spdx_is_native(d, recipe):
return any(a.annotationType == "OTHER" and
a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION")) and
@@ -408,12 +419,7 @@ python do_create_spdx() {
recipe.versionInfo = d.getVar("PV")
recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
if bb.data.inherits_class("native", d):
-annotation = oe.spdx.SPDXAnnotation()
-annotation.annotationDate = creation_time
-annotation.annotationType = "OTHER"
-annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION"))
-annotation.comment = "isNative"
-recipe.annotations.append(annotation)
+recipe.annotations.append(create_annotation(d, "isNative"))
for s in d.getVar('SRC_URI').split():
if not s.startswith("file://"):
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157458):
https://lists.openembedded.org/g/openembedded-core/message/157458
Mute This Topic: https://lists.openembedded.org/mt/86616598/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 3/3] create-spdx: cross recipes are native also
Recipes that inherit cross should also be categorized as isNative
Signed-off-by: Saul Wold
---
classes/create-spdx.bbclass | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/classes/create-spdx.bbclass b/classes/create-spdx.bbclass
index 72a9b55..883bc23 100644
--- a/classes/create-spdx.bbclass
+++ b/classes/create-spdx.bbclass
@@ -418,8 +418,9 @@ python do_create_spdx() {
recipe.name = d.getVar("PN")
recipe.versionInfo = d.getVar("PV")
recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
-if bb.data.inherits_class("native", d):
+if bb.data.inherits_class("native", d) or bb.data.inherits_class("cross",
d):
recipe.annotations.append(create_annotation(d, "isNative"))
+recipe.annotations.append(create_annotation(d, "SPDXDIR:%s" %
str(spdx_workdir).replace(str(top_dir) +'/', '')))
for s in d.getVar('SRC_URI').split():
if not s.startswith("file://"):
@@ -610,7 +611,7 @@ python do_create_runtime_spdx() {
deploy_dir_spdx = Path(d.getVar("DEPLOY_DIR_SPDX"))
spdx_deploy = Path(d.getVar("SPDXRUNTIMEDEPLOY"))
-is_native = bb.data.inherits_class("native", d)
+is_native = bb.data.inherits_class("native", d) or
bb.data.inherits_class("cross", d)
creation_time =
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
--
2.31.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#157460):
https://lists.openembedded.org/g/openembedded-core/message/157460
Mute This Topic: https://lists.openembedded.org/mt/86616600/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/3] spdx.py: Add annotation to relationship
Having annotations on relationship can provide additional information about the relationship such as how it was derived. Signed-off-by: Saul Wold --- lib/oe/spdx.py | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/oe/spdx.py b/lib/oe/spdx.py index 4416194..9e7ced5 100644 --- a/lib/oe/spdx.py +++ b/lib/oe/spdx.py @@ -196,6 +196,7 @@ class SPDXRelationship(SPDXObject): relatedSpdxElement = _String() relationshipType = _String() comment = _String() +annotations = _ObjectList(SPDXAnnotation) class SPDXExternalReference(SPDXObject): @@ -300,7 +301,7 @@ class SPDXDocument(SPDXObject): def from_json(cls, f): return cls(**json.load(f)) -def add_relationship(self, _from, relationship, _to, *, comment=None): +def add_relationship(self, _from, relationship, _to, *, comment=None, annotation=None): if isinstance(_from, SPDXObject): from_spdxid = _from.SPDXID else: @@ -320,6 +321,9 @@ class SPDXDocument(SPDXObject): if comment is not None: r.comment = comment +if annotation is not None: +r.annotations.append(annotation) + self.relationships.append(r) def find_by_spdxid(self, spdxid): -- 2.31.1 -=-=-=-=-=-=-=-=-=-=-=- Links: You receive all messages sent to this group. View/Reply Online (#157457): https://lists.openembedded.org/g/openembedded-core/message/157457 Mute This Topic: https://lists.openembedded.org/mt/86616597/21656 Group Owner: openembedded-core+ow...@lists.openembedded.org Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub [arch...@mail-archive.com] -=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 1/2] spdx.py: Add SPDXAnnotation Object
This is added to allow the create-spdx code to create annotations
that store values properly according to the SPDX Specification.
Initialy they will be used to track if a recipe is a native type.
Signed-off-by: Saul Wold
---
meta/lib/oe/spdx.py | 7 +++
1 file changed, 7 insertions(+)
diff --git a/meta/lib/oe/spdx.py b/meta/lib/oe/spdx.py
index 9814fbfd66..a99e54ff40 100644
--- a/meta/lib/oe/spdx.py
+++ b/meta/lib/oe/spdx.py
@@ -123,6 +123,12 @@ class SPDXObject(metaclass=MetaSPDXObject):
raise KeyError("%r is not a valid SPDX property" % name)
+class SPDXAnnotation(SPDXObject):
+annotationDate = _String()
+annotationType = _String()
+annotator = _String()
+comment = _String()
+
class SPDXChecksum(SPDXObject):
algorithm = _String()
checksumValue = _String()
@@ -164,6 +170,7 @@ class SPDXPackage(SPDXObject):
packageVerificationCode = _Object(SPDXPackageVerificationCode)
hasFiles = _StringList()
packageFileName = _String()
+annotations = _ObjectList(SPDXAnnotation)
class SPDXFile(SPDXObject):
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156393):
https://lists.openembedded.org/g/openembedded-core/message/156393
Mute This Topic: https://lists.openembedded.org/mt/85906171/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
[OE-core] [PATCH 2/2] create-spdx: Use SPDXAnnotation to track native recipes
Create a small function that checks for 'isNative' as part of an Annotation
When the collect_dep_sources() runs, it collects sources from both native
and non-native recipes. Later when the GENERATED_FROM matching occurs it
may find the file (via checksum) from the native recipe since it's the
same checksum as the target file. The that are generated DocumentRefs
point to the native recipe rather than the target recipe DocumentRef.
Signed-off-by: Saul Wold
---
meta/classes/create-spdx.bbclass | 21 +++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/meta/classes/create-spdx.bbclass b/meta/classes/create-spdx.bbclass
index 3c73c21c04..739b46e9b3 100644
--- a/meta/classes/create-spdx.bbclass
+++ b/meta/classes/create-spdx.bbclass
@@ -13,6 +13,9 @@ SPDXDIR ??= "${WORKDIR}/spdx"
SPDXDEPLOY = "${SPDXDIR}/deploy"
SPDXWORK = "${SPDXDIR}/work"
+SPDX_TOOL_NAME ??= "oe-spdx-creator"
+SPDX_TOOL_VERSION ??= "1.0"
+
SPDXRUNTIMEDEPLOY = "${SPDXDIR}/runtime-deploy"
SPDX_INCLUDE_SOURCES ??= "0"
@@ -32,6 +35,10 @@ def get_doc_namespace(d, doc):
namespace_uuid = uuid.uuid5(uuid.NAMESPACE_DNS,
d.getVar("SPDX_UUID_NAMESPACE"))
return "%s/%s-%s" % (d.getVar("SPDX_NAMESPACE_PREFIX"), doc.name,
str(uuid.uuid5(namespace_uuid, doc.name)))
+def recipe_spdx_is_native(d, recipe):
+return any(a.annotationType == "OTHER" and
+ a.annotator == "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION")) and
+ a.comment == "isNative" for a in recipe.annotations)
def is_work_shared(d):
pn = d.getVar('PN')
@@ -336,6 +343,10 @@ def collect_dep_sources(d, dep_recipes):
sources = {}
for dep in dep_recipes:
+# Don't collect sources from native recipes as they
+# match non-native sources also.
+if recipe_spdx_is_native(d, dep.recipe):
+continue
recipe_files = set(dep.recipe.hasFiles)
for spdx_file in dep.doc.files:
@@ -382,7 +393,6 @@ python do_create_spdx() {
include_sources = d.getVar("SPDX_INCLUDE_SOURCES") == "1"
archive_sources = d.getVar("SPDX_ARCHIVE_SOURCES") == "1"
archive_packaged = d.getVar("SPDX_ARCHIVE_PACKAGED") == "1"
-is_native = bb.data.inherits_class("native", d)
creation_time =
datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -401,6 +411,13 @@ python do_create_spdx() {
recipe.name = d.getVar("PN")
recipe.versionInfo = d.getVar("PV")
recipe.SPDXID = oe.sbom.get_recipe_spdxid(d)
+if bb.data.inherits_class("native", d):
+annotation = oe.spdx.SPDXAnnotation()
+annotation.annotationDate = creation_time
+annotation.annotationType = "OTHER"
+annotation.annotator = "Tool: %s - %s" % (d.getVar("SPDX_TOOL_NAME"),
d.getVar("SPDX_TOOL_VERSION"))
+annotation.comment = "isNative"
+recipe.annotations.append(annotation)
for s in d.getVar('SRC_URI').split():
if not s.startswith("file://"):
@@ -480,7 +497,7 @@ python do_create_spdx() {
sources = collect_dep_sources(d, dep_recipes)
found_licenses = {license.name:recipe_ref.externalDocumentId + ":" +
license.licenseId for license in doc.hasExtractedLicensingInfos}
-if not is_native:
+if not recipe_spdx_is_native(d, recipe):
bb.build.exec_func("read_subpackage_metadata", d)
pkgdest = Path(d.getVar("PKGDEST"))
--
2.25.1
-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#156394):
https://lists.openembedded.org/g/openembedded-core/message/156394
Mute This Topic: https://lists.openembedded.org/mt/85906172/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-
