Re: [OE-core] [PATCH v2] lib/spdx30_tasks: support directories deployed by image recipes

2025-01-15 Thread Igor Opaniuk via lists.openembedded.org 
On Mon, Jan 13, 2025 at 8:30 PM Joshua Watt  wrote:
>
> On Mon, Jan 13, 2025 at 10:51 AM  wrote:
> >
> > From: Igor Opaniuk 
> >
> > create_image_spdx() implementation assumes that image is indeed a file.
> > If image recipe deploys a directory (for example, which contains an
> > hierarchy of flash artifacts, that is used by SoC vendor-specific
> > flashing tool) which follows ${IMAGE_NAME}.${IMAGE_TYPE} naming scheme,
> > create_image_spdx() function will fail after trying to hash a directory:
> >
> > *** 0002:do_create_image_spdx(d)
> >  0003:
> > File: '.../meta/classes-recipe/create-spdx-image-3.0.bbclass', lineno: 48, 
> > function: do_create_image_spdx
> >  0044:addtask do_create_rootfs_spdx_setscene
> >  0045:
> >  0046:python do_create_image_spdx() {
> >  0047:import oe.spdx30_tasks
> >  *** 0048:oe.spdx30_tasks.create_image_spdx(d)
> >  0049:}
> >  0050:addtask do_create_image_spdx after do_image_complete 
> > do_create_rootfs_spdx before do_build
> >  0051:SSTATETASKS += "do_create_image_spdx"
> > ...
> > File: '.../bitbake/lib/bb/utils.py', lineno: 536, function: _hasher
> >  0532:
> >  0533:def _hasher(method, filename):
> >  0534:import mmap
> >  0535:
> >  *** 0536:with open(filename, "rb") as f:
> >  0537:try:
> >  0538:with mmap.mmap(f.fileno(), 0, 
> > access=mmap.ACCESS_READ) as mm:
> >  0539:for chunk in iter(lambda: mm.read(8192), b''):
> >  0540:method.update(chunk)
> > Exception: IsADirectoryError: [Errno 21] Is a directory: '...'
> >
> > Signed-off-by: Igor Opaniuk 
> > ---
> >  meta/lib/oe/spdx30_tasks.py | 51 -
> >  1 file changed, 34 insertions(+), 17 deletions(-)
> >
> > diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py
> > index c60c97896c..d1a7df5b64 100644
> > --- a/meta/lib/oe/spdx30_tasks.py
> > +++ b/meta/lib/oe/spdx30_tasks.py
> > @@ -1068,29 +1068,46 @@ def create_image_spdx(d):
> >  builds.append(image_build)
> >
> >  artifacts = []
> > +license_data = oe.spdx_common.load_spdx_license_data(d)
> >
> >  for image in task["images"]:
> >  image_filename = image["filename"]
> >  image_path = image_deploy_dir / image_filename
> > -a = objset.add_root(
> > -oe.spdx30.software_File(
> > -_id=objset.new_spdxid("image", image_filename),
> > -creationInfo=objset.doc.creationInfo,
> > -name=image_filename,
> > -verifiedUsing=[
> > -oe.spdx30.Hash(
> > -algorithm=oe.spdx30.HashAlgorithm.sha256,
> > -hashValue=bb.utils.sha256_file(image_path),
> > -)
> > -],
> > +if os.path.isdir(image_path):
> > +a = add_package_files(
> > +d,
> > +objset,
> > +image_path,
> > +lambda file_counter: objset.new_spdxid(
> > +"imagefile", str(file_counter)
> > +),
> > +lambda filepath: 
> > [oe.spdx30.software_SoftwarePurpose.file],
>
> Don't set the purpose here (e.g. just do `lambda filepath: []`), and
> set it later (see below)
>
> > +license_data,
>
> For now, lets just have add_package_files skip license scanning if
> license_data is None (specifically check for `None`), that way it
> matches the single image file below (which doesn't scan for licenses
> either).
>
> > +ignore_dirs=None,
> > +ignore_top_level_dirs=None,
> > +archive=None,
> >  )
>
>
>
> > -)
> > -set_purposes(
> > -d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, 
> > "SPDX_IMAGE_PURPOSE"
> > -)
> > -set_timestamp_now(d, a, "builtTime")
> > +artifacts.extend(a)
> > +else:
> > +a = objset.add_root(
> > +oe.spdx30.software_File(
> > +_id=objset.new_spdxid("image", image_filename),
> > +creationInfo=objset.doc.creationInfo,
> > +name=image_filename,
> > +verifiedUsing=[
> > +oe.spdx30.Hash(
> > +algorithm=oe.spdx30.HashAlgorithm.sha256,
> > +hashValue=bb.utils.sha256_file(image_path),
> > +)
> > +],
> > +)
> > +)
> > +set_purposes(
> > +d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, 
> > "SPDX_IMAGE_PURPOSE"
> > +)
> > +

Re: [OE-core] [PATCH v2] lib/spdx30_tasks: support directories deployed by image recipes

2025-01-13 Thread Joshua Watt via lists.openembedded.org 
On Mon, Jan 13, 2025 at 10:51 AM  wrote:
>
> From: Igor Opaniuk 
>
> create_image_spdx() implementation assumes that image is indeed a file.
> If image recipe deploys a directory (for example, which contains an
> hierarchy of flash artifacts, that is used by SoC vendor-specific
> flashing tool) which follows ${IMAGE_NAME}.${IMAGE_TYPE} naming scheme,
> create_image_spdx() function will fail after trying to hash a directory:
>
> *** 0002:do_create_image_spdx(d)
>  0003:
> File: '.../meta/classes-recipe/create-spdx-image-3.0.bbclass', lineno: 48, 
> function: do_create_image_spdx
>  0044:addtask do_create_rootfs_spdx_setscene
>  0045:
>  0046:python do_create_image_spdx() {
>  0047:import oe.spdx30_tasks
>  *** 0048:oe.spdx30_tasks.create_image_spdx(d)
>  0049:}
>  0050:addtask do_create_image_spdx after do_image_complete 
> do_create_rootfs_spdx before do_build
>  0051:SSTATETASKS += "do_create_image_spdx"
> ...
> File: '.../bitbake/lib/bb/utils.py', lineno: 536, function: _hasher
>  0532:
>  0533:def _hasher(method, filename):
>  0534:import mmap
>  0535:
>  *** 0536:with open(filename, "rb") as f:
>  0537:try:
>  0538:with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) 
> as mm:
>  0539:for chunk in iter(lambda: mm.read(8192), b''):
>  0540:method.update(chunk)
> Exception: IsADirectoryError: [Errno 21] Is a directory: '...'
>
> Signed-off-by: Igor Opaniuk 
> ---
>  meta/lib/oe/spdx30_tasks.py | 51 -
>  1 file changed, 34 insertions(+), 17 deletions(-)
>
> diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py
> index c60c97896c..d1a7df5b64 100644
> --- a/meta/lib/oe/spdx30_tasks.py
> +++ b/meta/lib/oe/spdx30_tasks.py
> @@ -1068,29 +1068,46 @@ def create_image_spdx(d):
>  builds.append(image_build)
>
>  artifacts = []
> +license_data = oe.spdx_common.load_spdx_license_data(d)
>
>  for image in task["images"]:
>  image_filename = image["filename"]
>  image_path = image_deploy_dir / image_filename
> -a = objset.add_root(
> -oe.spdx30.software_File(
> -_id=objset.new_spdxid("image", image_filename),
> -creationInfo=objset.doc.creationInfo,
> -name=image_filename,
> -verifiedUsing=[
> -oe.spdx30.Hash(
> -algorithm=oe.spdx30.HashAlgorithm.sha256,
> -hashValue=bb.utils.sha256_file(image_path),
> -)
> -],
> +if os.path.isdir(image_path):
> +a = add_package_files(
> +d,
> +objset,
> +image_path,
> +lambda file_counter: objset.new_spdxid(
> +"imagefile", str(file_counter)
> +),
> +lambda filepath: 
> [oe.spdx30.software_SoftwarePurpose.file],

Don't set the purpose here (e.g. just do `lambda filepath: []`), and
set it later (see below)

> +license_data,

For now, lets just have add_package_files skip license scanning if
license_data is None (specifically check for `None`), that way it
matches the single image file below (which doesn't scan for licenses
either).

> +ignore_dirs=None,
> +ignore_top_level_dirs=None,
> +archive=None,
>  )



> -)
> -set_purposes(
> -d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, 
> "SPDX_IMAGE_PURPOSE"
> -)
> -set_timestamp_now(d, a, "builtTime")
> +artifacts.extend(a)
> +else:
> +a = objset.add_root(
> +oe.spdx30.software_File(
> +_id=objset.new_spdxid("image", image_filename),
> +creationInfo=objset.doc.creationInfo,
> +name=image_filename,
> +verifiedUsing=[
> +oe.spdx30.Hash(
> +algorithm=oe.spdx30.HashAlgorithm.sha256,
> +hashValue=bb.utils.sha256_file(image_path),
> +)
> +],
> +)
> +)
> +set_purposes(
> +d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, 
> "SPDX_IMAGE_PURPOSE"
> +)
> +set_timestamp_now(d, a, "builtTime")
>
> -artifacts.append(a)
> +artifacts.append(a)

Looks like you are not setting the builtTime timestamps in the case of
a directory. However, I think this code can be simplifed to cover both
cases with:

 f

[OE-core] [PATCH v2] lib/spdx30_tasks: support directories deployed by image recipes

2025-01-13 Thread Igor Opaniuk via lists.openembedded.org 
From: Igor Opaniuk 

create_image_spdx() implementation assumes that image is indeed a file.
If image recipe deploys a directory (for example, which contains an
hierarchy of flash artifacts, that is used by SoC vendor-specific
flashing tool) which follows ${IMAGE_NAME}.${IMAGE_TYPE} naming scheme,
create_image_spdx() function will fail after trying to hash a directory:

*** 0002:do_create_image_spdx(d)
 0003:
File: '.../meta/classes-recipe/create-spdx-image-3.0.bbclass', lineno: 48, 
function: do_create_image_spdx
 0044:addtask do_create_rootfs_spdx_setscene
 0045:
 0046:python do_create_image_spdx() {
 0047:import oe.spdx30_tasks
 *** 0048:oe.spdx30_tasks.create_image_spdx(d)
 0049:}
 0050:addtask do_create_image_spdx after do_image_complete 
do_create_rootfs_spdx before do_build
 0051:SSTATETASKS += "do_create_image_spdx"
...
File: '.../bitbake/lib/bb/utils.py', lineno: 536, function: _hasher
 0532:
 0533:def _hasher(method, filename):
 0534:import mmap
 0535:
 *** 0536:with open(filename, "rb") as f:
 0537:try:
 0538:with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as 
mm:
 0539:for chunk in iter(lambda: mm.read(8192), b''):
 0540:method.update(chunk)
Exception: IsADirectoryError: [Errno 21] Is a directory: '...'

Signed-off-by: Igor Opaniuk 
---
 meta/lib/oe/spdx30_tasks.py | 51 -
 1 file changed, 34 insertions(+), 17 deletions(-)

diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py
index c60c97896c..d1a7df5b64 100644
--- a/meta/lib/oe/spdx30_tasks.py
+++ b/meta/lib/oe/spdx30_tasks.py
@@ -1068,29 +1068,46 @@ def create_image_spdx(d):
 builds.append(image_build)
 
 artifacts = []
+license_data = oe.spdx_common.load_spdx_license_data(d)
 
 for image in task["images"]:
 image_filename = image["filename"]
 image_path = image_deploy_dir / image_filename
-a = objset.add_root(
-oe.spdx30.software_File(
-_id=objset.new_spdxid("image", image_filename),
-creationInfo=objset.doc.creationInfo,
-name=image_filename,
-verifiedUsing=[
-oe.spdx30.Hash(
-algorithm=oe.spdx30.HashAlgorithm.sha256,
-hashValue=bb.utils.sha256_file(image_path),
-)
-],
+if os.path.isdir(image_path):
+a = add_package_files(
+d,
+objset,
+image_path,
+lambda file_counter: objset.new_spdxid(
+"imagefile", str(file_counter)
+),
+lambda filepath: 
[oe.spdx30.software_SoftwarePurpose.file],
+license_data,
+ignore_dirs=None,
+ignore_top_level_dirs=None,
+archive=None,
 )
-)
-set_purposes(
-d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, "SPDX_IMAGE_PURPOSE"
-)
-set_timestamp_now(d, a, "builtTime")
+artifacts.extend(a)
+else:
+a = objset.add_root(
+oe.spdx30.software_File(
+_id=objset.new_spdxid("image", image_filename),
+creationInfo=objset.doc.creationInfo,
+name=image_filename,
+verifiedUsing=[
+oe.spdx30.Hash(
+algorithm=oe.spdx30.HashAlgorithm.sha256,
+hashValue=bb.utils.sha256_file(image_path),
+)
+],
+)
+)
+set_purposes(
+d, a, "SPDX_IMAGE_PURPOSE:%s" % imagetype, 
"SPDX_IMAGE_PURPOSE"
+)
+set_timestamp_now(d, a, "builtTime")
 
-artifacts.append(a)
+artifacts.append(a)
 
 if artifacts:
 objset.new_scoped_relationship(
-- 
2.43.0


-=-=-=-=-=-=-=-=-=-=-=-
Links: You receive all messages sent to this group.
View/Reply Online (#209740): 
https://lists.openembedded.org/g/openembedded-core/message/209740
Mute This Topic: https://lists.openembedded.org/mt/110591862/21656
Group Owner: openembedded-core+ow...@lists.openembedded.org
Unsubscribe: https://lists.openembedded.org/g/openembedded-core/unsub 
[arch...@mail-archive.com]
-=-=-=-=-=-=-=-=-=-=-=-